def _write_lcmaps_file(self):
old_lcmaps_contents = utilities.read_file(LCMAPS_DB_LOCATION,
default='')
if old_lcmaps_contents and 'THIS FILE WAS WRITTEN BY OSG-CONFIGURE' not in old_lcmaps_contents:
backup_path = LCMAPS_DB_LOCATION + '.pre-configure'
self.log("Backing up %s to %s" % (LCMAPS_DB_LOCATION, backup_path),
level=logging.WARNING)
if not utilities.atomic_write(backup_path, old_lcmaps_contents):
msg = "Unable to back up old lcmaps.db to %s" % backup_path
self.log(msg, level=logging.ERROR)
raise exceptions.ConfigureError(msg)
self.log("Writing " + LCMAPS_DB_LOCATION, level=logging.INFO)
if self.authorization_method == 'xacml':
lcmaps_template_fn = 'lcmaps.db.gums'
elif self.authorization_method == 'gridmap' or self.authorization_method == 'local-gridmap':
lcmaps_template_fn = 'lcmaps.db.gridmap'
elif self.authorization_method == 'vomsmap':
if self.all_fqans:
lcmaps_template_fn = 'lcmaps.db.vomsmap.allfqans'
else:
lcmaps_template_fn = 'lcmaps.db.vomsmap'
else:
assert False
lcmaps_template_path = os.path.join(LCMAPS_DB_TEMPLATES_LOCATION,
lcmaps_template_fn)
if not validation.valid_file(lcmaps_template_path):
msg = "lcmaps.db template file not found at %s; ensure lcmaps-db-templates >= 1.6.6-1.8" \
" is installed or set edit_lcmaps_db=False" % lcmaps_template_path
self.log(msg, level=logging.ERROR)
raise exceptions.ConfigureError(msg)
old_lcmaps_contents = utilities.read_file(LCMAPS_DB_LOCATION,
default='')
if old_lcmaps_contents and 'THIS FILE WAS WRITTEN BY OSG-CONFIGURE' not in old_lcmaps_contents:
backup_path = LCMAPS_DB_LOCATION + '.pre-configure'
self.log("Backing up %s to %s" % (LCMAPS_DB_LOCATION, backup_path),
level=logging.WARNING)
try:
shutil.copy2(LCMAPS_DB_LOCATION, backup_path)
except EnvironmentError as err:
msg = "Unable to back up old lcmaps.db: " + str(err)
self.log(msg, level=logging.ERROR)
raise exceptions.ConfigureError(msg)
lcmaps_contents = utilities.read_file(lcmaps_template_path)
lcmaps_contents = (
"# THIS FILE WAS WRITTEN BY OSG-CONFIGURE AND WILL BE OVERWRITTEN ON FUTURE RUNS\n"
"# Set edit_lcmaps_db = False in the [%s] section of your OSG configuration to\n"
"# keep your changes.\n" % self.config_section +
lcmaps_contents.replace('@GUMSHOST@',
str(self.options['gums_host'].value)))
if not utilities.atomic_write(LCMAPS_DB_LOCATION, lcmaps_contents):
msg = "Error while writing to " + LCMAPS_DB_LOCATION
self.log(msg, level=logging.ERROR)
raise exceptions.ConfigureError(msg)
def configure(self, attributes):
"""Configure installation using attributes"""
self.log('MiscConfiguration.configure started')
if not self.enabled:
self.log('Not enabled')
self.log('MiscConfiguration.configure completed')
return True
# run fetch-crl script
if not utilities.fetch_crl():
self.log("Error while running fetch-crl script", level=logging.ERROR)
raise exceptions.ConfigureError('fetch-crl returned non-zero exit code')
using_gums = False
if self.options['authorization_method'].value == 'xacml':
using_gums = True
self._enable_xacml()
elif self.options['authorization_method'].value == 'gridmap':
self._disable_callout()
elif self.options['authorization_method'].value == 'local-gridmap':
self._disable_callout()
else:
self.log("Unknown authorization method: %s" % \
self.options['authorization_method'].value,
option='authorization_method',
section=self.config_section,
level=logging.ERROR)
raise exceptions.ConfigureError("Invalid authorization_method option " +
"in Misc Services")
if self.options['edit_lcmaps_db'].value:
if validation.valid_file(LCMAPS_DB_LOCATION):
self._update_lcmaps_file(using_gums)
else:
self.log("Not updating lcmaps.db because it's not accessible",
level=logging.DEBUG)
else:
self.log("Not updating lcmaps.db because edit_lcmaps_db is false",
level=logging.DEBUG)
if self.htcondor_gateway_enabled:
self.write_gridmap_to_htcondor_ce_config()
ensure_valid_user_vo_file(using_gums, logger=self.logger)
# Call configure_vdt_cleanup (enabling or disabling as necessary)
self._configure_cleanup()
self.log('MiscConfiguration.configure completed')
return True
def _subscribe_probe_to_remote_host(
self, probe, probe_file, remote_host, local_resource, local_host):
"""Subscribe the given probe to the given remote host if necessary --
this means:
- Enable the probe
- Set the local host name in the probe config (in ProbeName)
- Set the local resource name (in SiteName)
- Set the grid group (in Grid)
- Set the *Host settings to the the remote host
Check to see if a given probe has the correct subscription and if not
make it.
"""
self.log("GratiaConfiguration._subscribe_probe_to_remote_host started")
# XXX This just checks EnableProbe and SOAPHost; should we check the other *Host
# settings or are we using SOAPHost as a "don't configure me" sentinel?
# -mat 2/19/21
if self._subscription_present(probe_file, remote_host):
self.log("Subscription found %s probe, returning" % probe)
self.log("GratiaConfiguration._subscribe_probe_to_remote_host completed")
return True
if probe == 'gridftp':
probe = 'gridftp-transfer'
try:
buf = open(probe_file, "r", encoding="latin-1").read()
buf = self.replace_setting(buf, 'ProbeName', "%s:%s" % (probe, local_host))
buf = self.replace_setting(buf, 'SiteName', local_resource)
buf = self.replace_setting(buf, 'Grid', self.grid_group)
buf = self.replace_setting(buf, 'EnableProbe', '1')
for var in ['SSLHost', 'SOAPHost', 'SSLRegistrationHost', 'CollectorHost']:
buf = self.replace_setting(buf, var, remote_host)
if not utilities.atomic_write(probe_file, buf, mode=0o644):
self.log("Error while configuring gratia probes: " +
"can't write to %s" % probe_file,
level=logging.ERROR)
raise exceptions.ConfigureError("Error configuring gratia")
except OSError:
self.log("Error while configuring gratia probes",
exception=True,
level=logging.ERROR)
raise exceptions.ConfigureError("Error configuring gratia")
self.log("GratiaConfiguration._subscribe_probe_to_remote_host completed")
return True
def _make_subscription(self, probe, probe_file, probe_host, site,
hostname):
"""
Check to see if a given probe has the correct subscription and if not
make it.
"""
self.log("GratiaConfiguration._make_subscription started")
if self._subscription_present(probe_file, probe_host):
self.log("Subscription found %s probe, returning" % probe)
self.log("GratiaConfiguration._make_subscription completed")
return True
if probe == 'gridftp':
probe = 'gridftp-transfer'
try:
buf = open(probe_file).read()
buf = re.sub(r'(\s*)ProbeName\s*=.*',
r'\1ProbeName="' + "%s:%s" % (probe, hostname) + '"',
buf, 1)
buf = re.sub(r'(\s*)SiteName\s*=.*', r'\1SiteName="' + site + '"',
buf, 1)
buf = re.sub(r'(\s*)Grid\s*=.*',
r'\1Grid="' + self.grid_group + '"', buf, 1)
buf = re.sub(r'(\s*)EnableProbe\s*=.*', r'\1EnableProbe="1"', buf,
1)
for var in [
'SSLHost', 'SOAPHost', 'SSLRegistrationHost',
'CollectorHost'
]:
buf = re.sub(r'(\s*)' + var + r'\s*=.*',
r'\1' + var + '="' + probe_host + '"', buf, 1)
if not utilities.atomic_write(probe_file, buf, mode=420):
self.log("Error while configuring gratia probes: " +
"can't write to %s" % probe_file,
level=logging.ERROR)
raise exceptions.ConfigureError("Error configuring gratia")
except (IOError, OSError):
self.log("Error while configuring gratia probes",
exception=True,
level=logging.ERROR)
raise exceptions.ConfigureError("Error configuring gratia")
self.log("GratiaConfiguration._make_subscription completed")
return True
def _configure_srm_metrics(self):
""" Enable SRM metric """
if not self._srm_hosts:
self.log("No srm_hosts defined. Not configuring SRM metrics")
return
# Do some checking on the values. perhaps this should be in the validate section?
srm_dirs = split_list(self.options['srm_dir'].value)
if len(self._srm_hosts) != len(srm_dirs):
self.log("When enabling SRM metrics you must specify the same number " +
"of entries in the srm_dir variable as you have in the " +
"srm_hosts section. There are %i host entries and %i " \
"srm_dir entries." % (len(self._srm_hosts), len(srm_dirs)),
level=logging.ERROR)
raise exceptions.ConfigureError("Failed to configure RSV")
srm_ws_paths = []
if not utilities.blank(self.options['srm_webservice_path'].value):
srm_ws_paths = split_list(
self.options['srm_webservice_path'].value)
if len(self._srm_hosts) != len(srm_ws_paths):
self.log("If you set srm_webservice_path when enabling SRM metrics " +
"you must specify the same number of entries in the " +
"srm_webservice_path variable as you have in the srm_hosts " +
"section. There are %i host entries and %i " \
"srm_webservice_path entries." % (len(self._srm_hosts),
len(srm_ws_paths)),
level=logging.ERROR)
raise exceptions.ConfigureError("Failed to configure RSV")
# Now time to do the actual configuration
srm_metrics = self._get_metrics_by_type("OSG-SRM")
count = 0
for srm_host in self._srm_hosts:
self.log("Enabling SRM metrics for host '%s'" % srm_host)
args = ["--arg", "srm-destination-dir=%s" % srm_dirs[count]]
if srm_ws_paths:
args += [
"--arg",
"srm-webservice-path=%s" % srm_ws_paths[count]
]
self._enable_metrics(srm_host, srm_metrics, args)
count += 1
def ensure_valid_user_vo_file(using_gums, logger=utilities.NullLogger):
if not validation.valid_user_vo_file(USER_VO_MAP_LOCATION):
logger.info("Trying to create user-vo-map file")
result = create_user_vo_file(using_gums)
temp, invalid_lines = validation.valid_user_vo_file(USER_VO_MAP_LOCATION, True)
result = result and temp
if not result:
logger.error("Can't generate user-vo-map, manual intervention is needed")
if not invalid_lines:
logger.error("gums-host-cron or edg-mkgridmap generated an empty " +
USER_VO_MAP_LOCATION + " file, please check the "
"appropriate configuration and or log messages")
raise exceptions.ConfigureError('Error when generating user-vo-map file')
logger.error("Invalid lines in user-vo-map file:")
logger.error("\n".join(invalid_lines))
raise exceptions.ConfigureError("Error when invoking gums-host-cron or edg-mkgridmap")
def _configure_cleanup(self):
"""
Configure osg-cleanup
"""
# Do basic error checking to validate that this is a cron string
if len(re.split(r'\s+', self.options['cleanup_cron_time'].value)) != 5:
err_msg = "Error: the value of cleanup_cron_time must be a 5 part " \
"cron string: %s" % self.options['cleanup_cron_time'].value
self.log(err_msg,
option='cleanup_cron_time',
section=self.config_section,
level=logging.ERROR)
raise exceptions.ConfigureError(err_msg)
filehandle = open('/etc/osg/osg-cleanup.conf', 'w')
filehandle.write('# This file is automatically generated by osg-configure\n')
filehandle.write('# Manual modifications to this file may be overwritten\n')
filehandle.write('# Instead, modify /etc/osg/config.d/10-misc.ini\n')
filehandle.write('age = %s\n' % (self.options['cleanup_age_in_days'].value))
filehandle.write('users = %s\n' % (self.options['cleanup_users_list'].value))
filehandle.close()
# Writing this file seems a little hacky, but I'm not sure of a better way
filehandle = open('/etc/cron.d/osg-cleanup', 'w')
filehandle.write('%s root [ ! -f /var/lock/subsys/osg-cleanup-cron ] || /usr/sbin/osg-cleanup\n' %
(self.options['cleanup_cron_time'].value))
filehandle.close()
return True
def _enable_xacml(self):
"""
Enable authorization services using xacml protocol
"""
self.log("Updating " + GSI_AUTHZ_LOCATION, level=logging.INFO)
gsi_contents = "globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout\n"
if not utilities.atomic_write(GSI_AUTHZ_LOCATION, gsi_contents):
self.log("Error while writing to " + GSI_AUTHZ_LOCATION,
level=logging.ERROR)
raise exceptions.ConfigureError("Error while writing to " +
GSI_AUTHZ_LOCATION)
self.log("Updating " + GUMS_CLIENT_LOCATION, level=logging.INFO)
location_re = re.compile("^gums.location=.*$", re.MULTILINE)
authz_re = re.compile("^gums.authz=.*$", re.MULTILINE)
if not validation.valid_file(GUMS_CLIENT_LOCATION):
gums_properties = "gums.location=https://%s:8443" % (self.options['gums_host'].value)
gums_properties += "/gums/services/GUMSAdmin\n"
gums_properties += "gums.authz=https://%s:8443" % (self.options['gums_host'].value)
gums_properties += "/gums/services/GUMSXACMLAuthorizationServicePort"
else:
gums_properties = open(GUMS_CLIENT_LOCATION).read()
replacement = "gums.location=https://%s:8443" % (self.options['gums_host'].value)
replacement += "/gums/services/GUMSAdmin"
gums_properties = location_re.sub(replacement, gums_properties)
replacement = "gums.authz=https://%s:8443" % (self.options['gums_host'].value)
replacement += "/gums/services/GUMSXACMLAuthorizationServicePort"
gums_properties = authz_re.sub(replacement, gums_properties)
utilities.atomic_write(GUMS_CLIENT_LOCATION, gums_properties)
def configure(self, attributes):
"""
Configure installation using attributes.
"""
self.log('GipConfiguration.configure started')
if not self.enabled:
self.log('Not enabled, exiting...')
self.log('GipConfiguration.configure completed')
return
try:
gip_pwent = pwd.getpwnam(self.gip_user)
except KeyError, e:
if self.gip_user != 'tomcat':
self.gip_user = '******'
self.log("Couldn't find username %s, trying tomcat" %
self.gip_user,
exception=True,
level=logging.WARNING)
try:
gip_pwent = pwd.getpwnam(self.gip_user)
except KeyError, e:
self.log("Couldn't find username %s" % self.gip_user,
exception=True,
level=logging.ERROR)
raise exceptions.ConfigureError(
"Couldn't find username %s: %s" % (self.gip_user, e))
def _make_subscription(self, probe, probe_file, probe_host, site,
hostname):
"""
Check to see if a given probe has the correct subscription and if not
make it.
"""
self.log("GratiaConfiguration._make_subscription started")
if self._subscription_present(probe_file, probe_host):
self.log("Subscription found %s probe, returning" % probe)
self.log("GratiaConfiguration._make_subscription completed")
return True
if probe == 'gridftp':
probe = 'gridftp-transfer'
try:
buf = open(probe_file).read()
buf = self.replace_setting(buf, 'ProbeName',
"%s:%s" % (probe, hostname))
buf = self.replace_setting(buf, 'SiteName', site)
buf = self.replace_setting(buf, 'Grid', self.grid_group)
buf = self.replace_setting(buf, 'EnableProbe', '1')
for var in [
'SSLHost', 'SOAPHost', 'SSLRegistrationHost',
'CollectorHost'
]:
buf = self.replace_setting(buf, var, probe_host)
if not utilities.atomic_write(probe_file, buf, mode=420):
self.log("Error while configuring gratia probes: " +
"can't write to %s" % probe_file,
level=logging.ERROR)
raise exceptions.ConfigureError("Error configuring gratia")
except (IOError, OSError):
self.log("Error while configuring gratia probes",
exception=True,
level=logging.ERROR)
raise exceptions.ConfigureError("Error configuring gratia")
self.log("GratiaConfiguration._make_subscription completed")
return True
def _disable_callout(self):
"""
Enable authorization using gridmap files
"""
self.log("Updating " + GSI_AUTHZ_LOCATION, level=logging.INFO)
gsi_contents = "#globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout\n"
if not utilities.atomic_write(GSI_AUTHZ_LOCATION, gsi_contents):
self.log("Error while writing to " + GSI_AUTHZ_LOCATION,
level=logging.ERROR)
raise exceptions.ConfigureError("Error while writing to " +
GSI_AUTHZ_LOCATION)
def _set_lcmaps_callout(self, enable):
self.log("Updating " + GSI_AUTHZ_LOCATION, level=logging.INFO)
if enable:
gsi_contents = "globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout\n"
else:
gsi_contents = "#globus_mapping liblcas_lcmaps_gt4_mapping.so lcmaps_callout\n"
if not utilities.atomic_write(GSI_AUTHZ_LOCATION, gsi_contents):
msg = "Error while writing to " + GSI_AUTHZ_LOCATION
self.log(msg, level=logging.ERROR)
raise exceptions.ConfigureError(msg)
def configure(self, attributes):
"""Configure installation using attributes"""
self.log('MiscConfiguration.configure started')
if not self.enabled:
self.log('Not enabled')
self.log('MiscConfiguration.configure completed')
return True
# run fetch-crl script
if not utilities.fetch_crl():
self.log("Error while running fetch-crl script",
level=logging.ERROR)
raise exceptions.ConfigureError(
'fetch-crl returned non-zero exit code')
if self.authorization_method == 'xacml':
self._set_lcmaps_callout(True)
self.update_gums_client_location()
elif self.authorization_method == 'gridmap':
self._set_lcmaps_callout(False)
elif self.authorization_method == 'local-gridmap':
self._set_lcmaps_callout(False)
elif self.authorization_method == 'vomsmap':
self._set_lcmaps_callout(True)
else:
assert False, "Invalid authorization_method should have been caught in check_attributes()"
if self.options['edit_lcmaps_db'].value:
self._write_lcmaps_file()
else:
self.log("Not updating lcmaps.db because edit_lcmaps_db is false",
level=logging.DEBUG)
if self.htcondor_gateway_enabled and utilities.ce_installed():
self.write_gridmap_to_htcondor_ce_config()
self.log('MiscConfiguration.configure completed')
return True
def _configure_gridftp_metrics(self):
""" Enable GridFTP metrics for each GridFTP host declared """
if not self._gridftp_hosts:
self.log(
"No gridftp_hosts defined. Not configuring GridFTP metrics")
return
gridftp_dirs = split_list(self.options['gridftp_dir'].value)
if len(self._gridftp_hosts) != len(gridftp_dirs) and len(
gridftp_dirs) != 1:
self.log("RSV.gridftp_dir is set incorrectly. When enabling GridFTP " +
"metrics you must specify either exactly 1 entry, or the same " +
"number of entries in the gridftp_dir variable as you have in " +
"the gridftp_hosts section. There are %i host entries " \
"and %i gridftp_dir entries." % (len(self._gridftp_hosts),
len(gridftp_dirs)),
level=logging.ERROR)
raise exceptions.ConfigureError("Failed to configure RSV")
gridftp_metrics = self._get_metrics_by_type("GridFTP")
count = 0
for gridftp_host in self._gridftp_hosts:
self.log("Enabling GridFTP metrics for host '%s'" % gridftp_host)
if len(gridftp_dirs) == 1:
directories = gridftp_dirs[0]
else:
directories = gridftp_dirs[count]
args = ["--arg", "destination-dir=%s" % directories]
self._enable_metrics(gridftp_host, gridftp_metrics, args)
count += 1
def _update_lcmaps_text(self, lcmaps_db, gums, gums_host):
#
# Update GUMS endpoint (if using GUMS)
#
if gums:
endpoint_re = re.compile(r'^\s*"--endpoint\s+https://.*/gums/services.*"\s*?$',
re.MULTILINE)
replacement = " \"--endpoint https://%s:8443" % (gums_host)
replacement += "/gums/services/GUMSXACMLAuthorizationServicePort\""
lcmaps_db = endpoint_re.sub(replacement, lcmaps_db)
#
# Update "authorize_only" section
#
addition_comment = ("\n\n"
"## Added by osg-configure\n"
"## Set 'edit_lcmaps_db=False' in the [%s] section of your OSG configs\n"
"## to keep osg-configure from modifying this file\n" %
self.config_section)
# Split the string into four:
# 1. Everything before the authorize_only section
# 2. The header line to the authorize_only section
# 3. The body of the authorize_only section
# 4. Everything after the authorize_only section (if present)
#
# The authorize_only section ends at the end of the file (\Z) or when
# another section begins (^[ \t]*[a-zA-Z_]+:)
authorize_only_re = re.compile(r'\A(.+)(^[ \t]*authorize_only:[^\n]*?$)(.+?)(^[ \t]*[a-zA-Z_]+:.+|\Z)',
re.MULTILINE|re.DOTALL)
match = authorize_only_re.search(lcmaps_db)
if not match:
self.log("No valid 'authorize_only' section in lcmaps.db; cannot update!",
level=logging.ERROR)
raise exceptions.ConfigureError("No valid 'authorize_only' section in lcmaps.db")
pre_authorize_only, authorize_only_header, authorize_only, post_authorize_only = match.group(1, 2, 3, 4)
# Look for lines like
# "gumsclient -> good | bad" and
# "gridmapfile -> good | bad"
# which may be commented out.
gumsclient_re = re.compile(r'^\s*?[#]*?\s*?gumsclient\s*?->\s*?good\s*?[|]\s*?bad\s*?$',
re.MULTILINE)
gridmapfile_re = re.compile(r'^\s*?[#]*?\s*?gridmapfile\s*?->\s*?good\s*?[|]\s*?bad\s*?$',
re.MULTILINE)
if gums:
# Comment out gridmapfile line
authorize_only = gridmapfile_re.sub("#gridmapfile -> good | bad", authorize_only)
# If there's a gumsclient line, uncomment it. If not, add one to the beginning of the authorize_only section.
authorize_only, changed = gumsclient_re.subn("gumsclient -> good | bad", authorize_only, count=1)
if not changed:
authorize_only = addition_comment + "gumsclient -> good | bad\n\n" + authorize_only
self.log("Added 'gumsclient' authorization method to authorize_only section of %s" % LCMAPS_DB_LOCATION,
level=logging.WARNING)
else:
# Comment out gumsclient line
authorize_only = gumsclient_re.sub("#gumsclient -> good | bad", authorize_only)
# If there's a gridmapfile line, uncomment it. If not, add one to the beginning of the authorize_only section.
authorize_only, changed = gridmapfile_re.subn("gridmapfile -> good | bad", authorize_only, count=1)
if not changed:
authorize_only = addition_comment + "gridmapfile -> good | bad\n\n" + authorize_only
self.log("Added 'gridmapfile' authorization method to authorize_only section of %s" % LCMAPS_DB_LOCATION,
level=logging.WARNING)
return pre_authorize_only + authorize_only_header + authorize_only + post_authorize_only
self.gip_user,
exception=True,
level=logging.WARNING)
try:
gip_pwent = pwd.getpwnam(self.gip_user)
except KeyError, e:
self.log("Couldn't find username %s" % self.gip_user,
exception=True,
level=logging.ERROR)
raise exceptions.ConfigureError(
"Couldn't find username %s: %s" % (self.gip_user, e))
else:
self.log("Couldn't find username %s" % self.gip_user,
exception=True,
level=logging.ERROR)
raise exceptions.ConfigureError(
"Couldn't find username %s: %s" % (self.gip_user, e))
(gip_uid, gip_gid) = gip_pwent[2:4]
gip_tmpdir = os.path.join('/', 'var', 'tmp', 'gip')
gip_logdir = os.path.join('/', 'var', 'log', 'gip')
try:
if not os.path.exists(gip_tmpdir):
self.log("%s is not present, recreating" % gip_logdir)
os.mkdir(gip_tmpdir)
if not os.path.isdir(gip_tmpdir):
self.log("%s is not a directory, " % gip_tmpdir +
"please remove it and recreate it as a directory ",
level=logging.ERROR)
raise exceptions.ConfigureError(
"GIP tmp directory not setup: %s" % gip_tmpdir)