def _write_lcmaps_file(self): old_lcmaps_contents = utilities.read_file(LCMAPS_DB_LOCATION, default='') if old_lcmaps_contents and 'THIS FILE WAS WRITTEN BY OSG-CONFIGURE' not in old_lcmaps_contents: backup_path = LCMAPS_DB_LOCATION + '.pre-configure' self.log("Backing up %s to %s" % (LCMAPS_DB_LOCATION, backup_path), level=logging.WARNING) if not utilities.atomic_write(backup_path, old_lcmaps_contents): msg = "Unable to back up old lcmaps.db to %s" % backup_path self.log(msg, level=logging.ERROR) raise exceptions.ConfigureError(msg) self.log("Writing " + LCMAPS_DB_LOCATION, level=logging.INFO) if self.authorization_method == 'xacml': lcmaps_template_fn = 'lcmaps.db.gums' elif self.authorization_method == 'gridmap' or self.authorization_method == 'local-gridmap': lcmaps_template_fn = 'lcmaps.db.gridmap' elif self.authorization_method == 'vomsmap': if self.all_fqans: lcmaps_template_fn = 'lcmaps.db.vomsmap.allfqans' else: lcmaps_template_fn = 'lcmaps.db.vomsmap' else: assert False lcmaps_template_path = os.path.join(LCMAPS_DB_TEMPLATES_LOCATION, lcmaps_template_fn) if not validation.valid_file(lcmaps_template_path): msg = "lcmaps.db template file not found at %s; ensure lcmaps-db-templates >= 1.6.6-1.8" \ " is installed or set edit_lcmaps_db=False" % lcmaps_template_path self.log(msg, level=logging.ERROR) raise exceptions.ConfigureError(msg) old_lcmaps_contents = utilities.read_file(LCMAPS_DB_LOCATION, default='') if old_lcmaps_contents and 'THIS FILE WAS WRITTEN BY OSG-CONFIGURE' not in old_lcmaps_contents: backup_path = LCMAPS_DB_LOCATION + '.pre-configure' self.log("Backing up %s to %s" % (LCMAPS_DB_LOCATION, backup_path), level=logging.WARNING) try: shutil.copy2(LCMAPS_DB_LOCATION, backup_path) except EnvironmentError as err: msg = "Unable to back up old lcmaps.db: " + str(err) self.log(msg, level=logging.ERROR) raise exceptions.ConfigureError(msg) lcmaps_contents = utilities.read_file(lcmaps_template_path) lcmaps_contents = ( "# THIS FILE WAS WRITTEN BY OSG-CONFIGURE AND WILL BE OVERWRITTEN ON FUTURE RUNS\n" "# Set edit_lcmaps_db = False in the [%s] section of your OSG configuration to\n" "# keep your changes.\n" % self.config_section + lcmaps_contents.replace('@GUMSHOST@', str(self.options['gums_host'].value))) if not utilities.atomic_write(LCMAPS_DB_LOCATION, lcmaps_contents): msg = "Error while writing to " + LCMAPS_DB_LOCATION self.log(msg, level=logging.ERROR) raise exceptions.ConfigureError(msg)
def configure(self, attributes): """Configure installation using attributes""" self.log('MiscConfiguration.configure started') if not self.enabled: self.log('Not enabled') self.log('MiscConfiguration.configure completed') return True # run fetch-crl script if not utilities.fetch_crl(): self.log("Error while running fetch-crl script", level=logging.ERROR) raise exceptions.ConfigureError('fetch-crl returned non-zero exit code') using_gums = False if self.options['authorization_method'].value == 'xacml': using_gums = True self._enable_xacml() elif self.options['authorization_method'].value == 'gridmap': self._disable_callout() elif self.options['authorization_method'].value == 'local-gridmap': self._disable_callout() else: self.log("Unknown authorization method: %s" % \ self.options['authorization_method'].value, option='authorization_method', section=self.config_section, level=logging.ERROR) raise exceptions.ConfigureError("Invalid authorization_method option " + "in Misc Services") if self.options['edit_lcmaps_db'].value: if validation.valid_file(LCMAPS_DB_LOCATION): self._update_lcmaps_file(using_gums) else: self.log("Not updating lcmaps.db because it's not accessible", level=logging.DEBUG) else: self.log("Not updating lcmaps.db because edit_lcmaps_db is false", level=logging.DEBUG) if self.htcondor_gateway_enabled: self.write_gridmap_to_htcondor_ce_config() ensure_valid_user_vo_file(using_gums, logger=self.logger) # Call configure_vdt_cleanup (enabling or disabling as necessary) self._configure_cleanup() self.log('MiscConfiguration.configure completed') return True
def _subscribe_probe_to_remote_host( self, probe, probe_file, remote_host, local_resource, local_host): """Subscribe the given probe to the given remote host if necessary -- this means: - Enable the probe - Set the local host name in the probe config (in ProbeName) - Set the local resource name (in SiteName) - Set the grid group (in Grid) - Set the *Host settings to the the remote host Check to see if a given probe has the correct subscription and if not make it. """ self.log("GratiaConfiguration._subscribe_probe_to_remote_host started") # XXX This just checks EnableProbe and SOAPHost; should we check the other *Host # settings or are we using SOAPHost as a "don't configure me" sentinel? # -mat 2/19/21 if self._subscription_present(probe_file, remote_host): self.log("Subscription found %s probe, returning" % probe) self.log("GratiaConfiguration._subscribe_probe_to_remote_host completed") return True if probe == 'gridftp': probe = 'gridftp-transfer' try: buf = open(probe_file, "r", encoding="latin-1").read() buf = self.replace_setting(buf, 'ProbeName', "%s:%s" % (probe, local_host)) buf = self.replace_setting(buf, 'SiteName', local_resource) buf = self.replace_setting(buf, 'Grid', self.grid_group) buf = self.replace_setting(buf, 'EnableProbe', '1') for var in ['SSLHost', 'SOAPHost', 'SSLRegistrationHost', 'CollectorHost']: buf = self.replace_setting(buf, var, remote_host) if not utilities.atomic_write(probe_file, buf, mode=0o644): self.log("Error while configuring gratia probes: " + "can't write to %s" % probe_file, level=logging.ERROR) raise exceptions.ConfigureError("Error configuring gratia") except OSError: self.log("Error while configuring gratia probes", exception=True, level=logging.ERROR) raise exceptions.ConfigureError("Error configuring gratia") self.log("GratiaConfiguration._subscribe_probe_to_remote_host completed") return True
def _make_subscription(self, probe, probe_file, probe_host, site, hostname): """ Check to see if a given probe has the correct subscription and if not make it. """ self.log("GratiaConfiguration._make_subscription started") if self._subscription_present(probe_file, probe_host): self.log("Subscription found %s probe, returning" % probe) self.log("GratiaConfiguration._make_subscription completed") return True if probe == 'gridftp': probe = 'gridftp-transfer' try: buf = open(probe_file).read() buf = re.sub(r'(\s*)ProbeName\s*=.*', r'\1ProbeName="' + "%s:%s" % (probe, hostname) + '"', buf, 1) buf = re.sub(r'(\s*)SiteName\s*=.*', r'\1SiteName="' + site + '"', buf, 1) buf = re.sub(r'(\s*)Grid\s*=.*', r'\1Grid="' + self.grid_group + '"', buf, 1) buf = re.sub(r'(\s*)EnableProbe\s*=.*', r'\1EnableProbe="1"', buf, 1) for var in [ 'SSLHost', 'SOAPHost', 'SSLRegistrationHost', 'CollectorHost' ]: buf = re.sub(r'(\s*)' + var + r'\s*=.*', r'\1' + var + '="' + probe_host + '"', buf, 1) if not utilities.atomic_write(probe_file, buf, mode=420): self.log("Error while configuring gratia probes: " + "can't write to %s" % probe_file, level=logging.ERROR) raise exceptions.ConfigureError("Error configuring gratia") except (IOError, OSError): self.log("Error while configuring gratia probes", exception=True, level=logging.ERROR) raise exceptions.ConfigureError("Error configuring gratia") self.log("GratiaConfiguration._make_subscription completed") return True
def _configure_srm_metrics(self): """ Enable SRM metric """ if not self._srm_hosts: self.log("No srm_hosts defined. Not configuring SRM metrics") return # Do some checking on the values. perhaps this should be in the validate section? srm_dirs = split_list(self.options['srm_dir'].value) if len(self._srm_hosts) != len(srm_dirs): self.log("When enabling SRM metrics you must specify the same number " + "of entries in the srm_dir variable as you have in the " + "srm_hosts section. There are %i host entries and %i " \ "srm_dir entries." % (len(self._srm_hosts), len(srm_dirs)), level=logging.ERROR) raise exceptions.ConfigureError("Failed to configure RSV") srm_ws_paths = [] if not utilities.blank(self.options['srm_webservice_path'].value): srm_ws_paths = split_list( self.options['srm_webservice_path'].value) if len(self._srm_hosts) != len(srm_ws_paths): self.log("If you set srm_webservice_path when enabling SRM metrics " + "you must specify the same number of entries in the " + "srm_webservice_path variable as you have in the srm_hosts " + "section. There are %i host entries and %i " \ "srm_webservice_path entries." % (len(self._srm_hosts), len(srm_ws_paths)), level=logging.ERROR) raise exceptions.ConfigureError("Failed to configure RSV") # Now time to do the actual configuration srm_metrics = self._get_metrics_by_type("OSG-SRM") count = 0 for srm_host in self._srm_hosts: self.log("Enabling SRM metrics for host '%s'" % srm_host) args = ["--arg", "srm-destination-dir=%s" % srm_dirs[count]] if srm_ws_paths: args += [ "--arg", "srm-webservice-path=%s" % srm_ws_paths[count] ] self._enable_metrics(srm_host, srm_metrics, args) count += 1
def ensure_valid_user_vo_file(using_gums, logger=utilities.NullLogger): if not validation.valid_user_vo_file(USER_VO_MAP_LOCATION): logger.info("Trying to create user-vo-map file") result = create_user_vo_file(using_gums) temp, invalid_lines = validation.valid_user_vo_file(USER_VO_MAP_LOCATION, True) result = result and temp if not result: logger.error("Can't generate user-vo-map, manual intervention is needed") if not invalid_lines: logger.error("gums-host-cron or edg-mkgridmap generated an empty " + USER_VO_MAP_LOCATION + " file, please check the " "appropriate configuration and or log messages") raise exceptions.ConfigureError('Error when generating user-vo-map file') logger.error("Invalid lines in user-vo-map file:") logger.error("\n".join(invalid_lines)) raise exceptions.ConfigureError("Error when invoking gums-host-cron or edg-mkgridmap")
def _configure_cleanup(self): """ Configure osg-cleanup """ # Do basic error checking to validate that this is a cron string if len(re.split(r'\s+', self.options['cleanup_cron_time'].value)) != 5: err_msg = "Error: the value of cleanup_cron_time must be a 5 part " \ "cron string: %s" % self.options['cleanup_cron_time'].value self.log(err_msg, option='cleanup_cron_time', section=self.config_section, level=logging.ERROR) raise exceptions.ConfigureError(err_msg) filehandle = open('/etc/osg/osg-cleanup.conf', 'w') filehandle.write('# This file is automatically generated by osg-configure\n') filehandle.write('# Manual modifications to this file may be overwritten\n') filehandle.write('# Instead, modify /etc/osg/config.d/10-misc.ini\n') filehandle.write('age = %s\n' % (self.options['cleanup_age_in_days'].value)) filehandle.write('users = %s\n' % (self.options['cleanup_users_list'].value)) filehandle.close() # Writing this file seems a little hacky, but I'm not sure of a better way filehandle = open('/etc/cron.d/osg-cleanup', 'w') filehandle.write('%s root [ ! -f /var/lock/subsys/osg-cleanup-cron ] || /usr/sbin/osg-cleanup\n' % (self.options['cleanup_cron_time'].value)) filehandle.close() return True
def _enable_xacml(self): """ Enable authorization services using xacml protocol """ self.log("Updating " + GSI_AUTHZ_LOCATION, level=logging.INFO) gsi_contents = "globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout\n" if not utilities.atomic_write(GSI_AUTHZ_LOCATION, gsi_contents): self.log("Error while writing to " + GSI_AUTHZ_LOCATION, level=logging.ERROR) raise exceptions.ConfigureError("Error while writing to " + GSI_AUTHZ_LOCATION) self.log("Updating " + GUMS_CLIENT_LOCATION, level=logging.INFO) location_re = re.compile("^gums.location=.*$", re.MULTILINE) authz_re = re.compile("^gums.authz=.*$", re.MULTILINE) if not validation.valid_file(GUMS_CLIENT_LOCATION): gums_properties = "gums.location=https://%s:8443" % (self.options['gums_host'].value) gums_properties += "/gums/services/GUMSAdmin\n" gums_properties += "gums.authz=https://%s:8443" % (self.options['gums_host'].value) gums_properties += "/gums/services/GUMSXACMLAuthorizationServicePort" else: gums_properties = open(GUMS_CLIENT_LOCATION).read() replacement = "gums.location=https://%s:8443" % (self.options['gums_host'].value) replacement += "/gums/services/GUMSAdmin" gums_properties = location_re.sub(replacement, gums_properties) replacement = "gums.authz=https://%s:8443" % (self.options['gums_host'].value) replacement += "/gums/services/GUMSXACMLAuthorizationServicePort" gums_properties = authz_re.sub(replacement, gums_properties) utilities.atomic_write(GUMS_CLIENT_LOCATION, gums_properties)
def configure(self, attributes): """ Configure installation using attributes. """ self.log('GipConfiguration.configure started') if not self.enabled: self.log('Not enabled, exiting...') self.log('GipConfiguration.configure completed') return try: gip_pwent = pwd.getpwnam(self.gip_user) except KeyError, e: if self.gip_user != 'tomcat': self.gip_user = '******' self.log("Couldn't find username %s, trying tomcat" % self.gip_user, exception=True, level=logging.WARNING) try: gip_pwent = pwd.getpwnam(self.gip_user) except KeyError, e: self.log("Couldn't find username %s" % self.gip_user, exception=True, level=logging.ERROR) raise exceptions.ConfigureError( "Couldn't find username %s: %s" % (self.gip_user, e))
def _make_subscription(self, probe, probe_file, probe_host, site, hostname): """ Check to see if a given probe has the correct subscription and if not make it. """ self.log("GratiaConfiguration._make_subscription started") if self._subscription_present(probe_file, probe_host): self.log("Subscription found %s probe, returning" % probe) self.log("GratiaConfiguration._make_subscription completed") return True if probe == 'gridftp': probe = 'gridftp-transfer' try: buf = open(probe_file).read() buf = self.replace_setting(buf, 'ProbeName', "%s:%s" % (probe, hostname)) buf = self.replace_setting(buf, 'SiteName', site) buf = self.replace_setting(buf, 'Grid', self.grid_group) buf = self.replace_setting(buf, 'EnableProbe', '1') for var in [ 'SSLHost', 'SOAPHost', 'SSLRegistrationHost', 'CollectorHost' ]: buf = self.replace_setting(buf, var, probe_host) if not utilities.atomic_write(probe_file, buf, mode=420): self.log("Error while configuring gratia probes: " + "can't write to %s" % probe_file, level=logging.ERROR) raise exceptions.ConfigureError("Error configuring gratia") except (IOError, OSError): self.log("Error while configuring gratia probes", exception=True, level=logging.ERROR) raise exceptions.ConfigureError("Error configuring gratia") self.log("GratiaConfiguration._make_subscription completed") return True
def _disable_callout(self): """ Enable authorization using gridmap files """ self.log("Updating " + GSI_AUTHZ_LOCATION, level=logging.INFO) gsi_contents = "#globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout\n" if not utilities.atomic_write(GSI_AUTHZ_LOCATION, gsi_contents): self.log("Error while writing to " + GSI_AUTHZ_LOCATION, level=logging.ERROR) raise exceptions.ConfigureError("Error while writing to " + GSI_AUTHZ_LOCATION)
def _set_lcmaps_callout(self, enable): self.log("Updating " + GSI_AUTHZ_LOCATION, level=logging.INFO) if enable: gsi_contents = "globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout\n" else: gsi_contents = "#globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout\n" if not utilities.atomic_write(GSI_AUTHZ_LOCATION, gsi_contents): msg = "Error while writing to " + GSI_AUTHZ_LOCATION self.log(msg, level=logging.ERROR) raise exceptions.ConfigureError(msg)
def configure(self, attributes): """Configure installation using attributes""" self.log('MiscConfiguration.configure started') if not self.enabled: self.log('Not enabled') self.log('MiscConfiguration.configure completed') return True # run fetch-crl script if not utilities.fetch_crl(): self.log("Error while running fetch-crl script", level=logging.ERROR) raise exceptions.ConfigureError( 'fetch-crl returned non-zero exit code') if self.authorization_method == 'xacml': self._set_lcmaps_callout(True) self.update_gums_client_location() elif self.authorization_method == 'gridmap': self._set_lcmaps_callout(False) elif self.authorization_method == 'local-gridmap': self._set_lcmaps_callout(False) elif self.authorization_method == 'vomsmap': self._set_lcmaps_callout(True) else: assert False, "Invalid authorization_method should have been caught in check_attributes()" if self.options['edit_lcmaps_db'].value: self._write_lcmaps_file() else: self.log("Not updating lcmaps.db because edit_lcmaps_db is false", level=logging.DEBUG) if self.htcondor_gateway_enabled and utilities.ce_installed(): self.write_gridmap_to_htcondor_ce_config() self.log('MiscConfiguration.configure completed') return True
def _configure_gridftp_metrics(self): """ Enable GridFTP metrics for each GridFTP host declared """ if not self._gridftp_hosts: self.log( "No gridftp_hosts defined. Not configuring GridFTP metrics") return gridftp_dirs = split_list(self.options['gridftp_dir'].value) if len(self._gridftp_hosts) != len(gridftp_dirs) and len( gridftp_dirs) != 1: self.log("RSV.gridftp_dir is set incorrectly. When enabling GridFTP " + "metrics you must specify either exactly 1 entry, or the same " + "number of entries in the gridftp_dir variable as you have in " + "the gridftp_hosts section. There are %i host entries " \ "and %i gridftp_dir entries." % (len(self._gridftp_hosts), len(gridftp_dirs)), level=logging.ERROR) raise exceptions.ConfigureError("Failed to configure RSV") gridftp_metrics = self._get_metrics_by_type("GridFTP") count = 0 for gridftp_host in self._gridftp_hosts: self.log("Enabling GridFTP metrics for host '%s'" % gridftp_host) if len(gridftp_dirs) == 1: directories = gridftp_dirs[0] else: directories = gridftp_dirs[count] args = ["--arg", "destination-dir=%s" % directories] self._enable_metrics(gridftp_host, gridftp_metrics, args) count += 1
def _update_lcmaps_text(self, lcmaps_db, gums, gums_host): # # Update GUMS endpoint (if using GUMS) # if gums: endpoint_re = re.compile(r'^\s*"--endpoint\s+https://.*/gums/services.*"\s*?$', re.MULTILINE) replacement = " \"--endpoint https://%s:8443" % (gums_host) replacement += "/gums/services/GUMSXACMLAuthorizationServicePort\"" lcmaps_db = endpoint_re.sub(replacement, lcmaps_db) # # Update "authorize_only" section # addition_comment = ("\n\n" "## Added by osg-configure\n" "## Set 'edit_lcmaps_db=False' in the [%s] section of your OSG configs\n" "## to keep osg-configure from modifying this file\n" % self.config_section) # Split the string into four: # 1. Everything before the authorize_only section # 2. The header line to the authorize_only section # 3. The body of the authorize_only section # 4. Everything after the authorize_only section (if present) # # The authorize_only section ends at the end of the file (\Z) or when # another section begins (^[ \t]*[a-zA-Z_]+:) authorize_only_re = re.compile(r'\A(.+)(^[ \t]*authorize_only:[^\n]*?$)(.+?)(^[ \t]*[a-zA-Z_]+:.+|\Z)', re.MULTILINE|re.DOTALL) match = authorize_only_re.search(lcmaps_db) if not match: self.log("No valid 'authorize_only' section in lcmaps.db; cannot update!", level=logging.ERROR) raise exceptions.ConfigureError("No valid 'authorize_only' section in lcmaps.db") pre_authorize_only, authorize_only_header, authorize_only, post_authorize_only = match.group(1, 2, 3, 4) # Look for lines like # "gumsclient -> good | bad" and # "gridmapfile -> good | bad" # which may be commented out. gumsclient_re = re.compile(r'^\s*?[#]*?\s*?gumsclient\s*?->\s*?good\s*?[|]\s*?bad\s*?$', re.MULTILINE) gridmapfile_re = re.compile(r'^\s*?[#]*?\s*?gridmapfile\s*?->\s*?good\s*?[|]\s*?bad\s*?$', re.MULTILINE) if gums: # Comment out gridmapfile line authorize_only = gridmapfile_re.sub("#gridmapfile -> good | bad", authorize_only) # If there's a gumsclient line, uncomment it. If not, add one to the beginning of the authorize_only section. authorize_only, changed = gumsclient_re.subn("gumsclient -> good | bad", authorize_only, count=1) if not changed: authorize_only = addition_comment + "gumsclient -> good | bad\n\n" + authorize_only self.log("Added 'gumsclient' authorization method to authorize_only section of %s" % LCMAPS_DB_LOCATION, level=logging.WARNING) else: # Comment out gumsclient line authorize_only = gumsclient_re.sub("#gumsclient -> good | bad", authorize_only) # If there's a gridmapfile line, uncomment it. If not, add one to the beginning of the authorize_only section. authorize_only, changed = gridmapfile_re.subn("gridmapfile -> good | bad", authorize_only, count=1) if not changed: authorize_only = addition_comment + "gridmapfile -> good | bad\n\n" + authorize_only self.log("Added 'gridmapfile' authorization method to authorize_only section of %s" % LCMAPS_DB_LOCATION, level=logging.WARNING) return pre_authorize_only + authorize_only_header + authorize_only + post_authorize_only
self.gip_user, exception=True, level=logging.WARNING) try: gip_pwent = pwd.getpwnam(self.gip_user) except KeyError, e: self.log("Couldn't find username %s" % self.gip_user, exception=True, level=logging.ERROR) raise exceptions.ConfigureError( "Couldn't find username %s: %s" % (self.gip_user, e)) else: self.log("Couldn't find username %s" % self.gip_user, exception=True, level=logging.ERROR) raise exceptions.ConfigureError( "Couldn't find username %s: %s" % (self.gip_user, e)) (gip_uid, gip_gid) = gip_pwent[2:4] gip_tmpdir = os.path.join('/', 'var', 'tmp', 'gip') gip_logdir = os.path.join('/', 'var', 'log', 'gip') try: if not os.path.exists(gip_tmpdir): self.log("%s is not present, recreating" % gip_logdir) os.mkdir(gip_tmpdir) if not os.path.isdir(gip_tmpdir): self.log("%s is not a directory, " % gip_tmpdir + "please remove it and recreate it as a directory ", level=logging.ERROR) raise exceptions.ConfigureError( "GIP tmp directory not setup: %s" % gip_tmpdir)