def setup_gram_config(self):
"""
Populate the gram config file with correct values
Returns True if successful, False otherwise
"""
buf = open(LSFConfiguration.GRAM_CONFIG_FILE).read()
for binfile in ['bsub', 'bqueues', 'bjobs', 'bhist', 'bacct', 'bkill']:
bin_location = os.path.join(self.lsf_bin_location, binfile)
if validation.valid_file(bin_location):
buf = utilities.add_or_replace_setting(buf, binfile, 'bin_location')
if self.options['seg_enabled'].value:
if (self.options['log_directory'].value is None or
not validation.valid_directory(self.options['log_directory'].value)):
mesg = "%s is not a valid directory location " % self.options['log_directory'].value
mesg += "for lsf log files"
self.log(mesg,
section=self.config_section,
option='log_directory',
level=logging.ERROR)
return False
buf = utilities.add_or_replace_setting(buf, 'log_path', self.options['log_directory'].value)
buf = utilities.add_or_replace_setting(buf, 'lsf_profile', self.options['lsf_profile'].value)
if not utilities.atomic_write(LSFConfiguration.GRAM_CONFIG_FILE, buf):
return False
return True
def setup_gram_config(self):
"""
Populate the gram config file with correct values
Returns True if successful, False otherwise
"""
buf = open(LSFConfiguration.GRAM_CONFIG_FILE).read()
for binfile in ['bsub', 'bqueues', 'bjobs', 'bhist', 'bacct', 'bkill']:
bin_location = os.path.join(self.lsf_bin_location, binfile)
if validation.valid_file(bin_location):
buf = utilities.add_or_replace_setting(buf, binfile,
'bin_location')
if self.options['seg_enabled'].value:
if (self.options['log_directory'].value is None
or not validation.valid_directory(
self.options['log_directory'].value)):
mesg = "%s is not a valid directory location " % self.options[
'log_directory'].value
mesg += "for lsf log files"
self.log(mesg,
section=self.config_section,
option='log_directory',
level=logging.ERROR)
return False
buf = utilities.add_or_replace_setting(
buf, 'log_path', self.options['log_directory'].value)
buf = utilities.add_or_replace_setting(
buf, 'lsf_profile', self.options['lsf_profile'].value)
if not utilities.atomic_write(LSFConfiguration.GRAM_CONFIG_FILE, buf):
return False
return True
def setup_gram_config(self):
"""
Populate the gram config file with correct values
Returns True if successful, False otherwise
"""
contents = open(PBSConfiguration.GRAM_CONFIG_FILE).read()
for binfile in ['qsub', 'qstat', 'qdel']:
bin_location = os.path.join(self.pbs_bin_location, binfile)
if validation.valid_file(bin_location):
contents = utilities.add_or_replace_setting(contents, binfile, bin_location)
if self.options['pbs_server'].value != '':
contents = utilities.add_or_replace_setting(contents, 'pbs_default', self.options['pbs_server'].value)
if self.options['seg_enabled'].value:
if (self.options['log_directory'].value is None or
not validation.valid_directory(self.options['log_directory'].value)):
mesg = "%s is not a valid directory location " % self.options['log_directory'].value
mesg += "for pbs log files"
self.log(mesg,
section=self.config_section,
option='log_directory',
level=logging.ERROR)
return False
contents = utilities.add_or_replace_setting(contents, 'log_path', self.options['log_directory'].value)
if not utilities.atomic_write(PBSConfiguration.GRAM_CONFIG_FILE, contents):
return False
return True
def _write_route_config_vars(self):
"""
Write condor-ce config attributes for the bosco job route. Sets values for:
- BOSCO_RMS
- BOSCO_ENDPOINT
"""
contents = utilities.read_file(self.HTCONDOR_CE_CONFIG_FILE,
default="# This file is managed by osg-configure\n")
contents = utilities.add_or_replace_setting(contents, "BOSCO_RMS", self.options['batch'].value,
quote_value=False)
contents = utilities.add_or_replace_setting(contents, "BOSCO_ENDPOINT", self.options['endpoint'].value,
quote_value=False)
utilities.atomic_write(self.HTCONDOR_CE_CONFIG_FILE, contents)
def setup_blah_config(self):
"""
Populate blah.config with correct values
Return True if successful, False otherwise
"""
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
contents = utilities.add_or_replace_setting(contents, "sge_rootpath", self.options['sge_root'].value,
quote_value=True)
contents = utilities.add_or_replace_setting(contents, "sge_cellname", self.options['sge_cell'].value,
quote_value=True)
return utilities.atomic_write(self.BLAH_CONFIG, contents)
return False
def setup_htcondor_ce_config(self):
"""
Populate the config file that tells htcondor-ce where the condor
pool is and where the spool directory is.
Returns True if successful, False otherwise
"""
if not utilities.rpm_installed('htcondor-ce'):
self.log("Unable to configure htcondor-ce for Condor: htcondor-ce not installed", level=logging.ERROR)
return False
def get_condor_ce_config_val(variable):
return utilities.get_condor_config_val(variable, executable='condor_ce_config_val', quiet_undefined=True)
# Get values for the settings we want to update. We can get the
# values from condor_config_val; in the case of JOB_ROUTER_SCHEDD2_NAME,
# we have FULL_HOSTNAME as a fallback in case SCHEDD_NAME is missing.
# We also get the current / default value from condor_ce_config_val;
# only update the setting in case the value from
# condor_config_val is different from the value from condor_ce_config_val.
condor_ce_config = {}
for condor_ce_config_key, condor_config_keys in [
('JOB_ROUTER_SCHEDD2_NAME', ['SCHEDD_NAME', 'FULL_HOSTNAME']),
('JOB_ROUTER_SCHEDD2_POOL', ['COLLECTOR_HOST']),
('JOB_ROUTER_SCHEDD2_SPOOL', ['SPOOL'])]:
condor_config_value = None
for condor_config_value in (utilities.get_condor_config_val(k, quiet_undefined=True) for k in
condor_config_keys):
if condor_config_value:
break
condor_ce_config_value = get_condor_ce_config_val(condor_ce_config_key)
if not (condor_config_value or condor_ce_config_value):
self.log("Unable to determine value for %s from %s and default not set; check your Condor config" %
(condor_ce_config_key, ' or '.join(condor_config_keys)), level=logging.ERROR)
return False
elif not condor_config_value:
continue # can't set anything for this
# Special case for JOB_ROUTER_SCHEDD2_POOL: append port if necessary (SOFTWARE-1744)
if condor_ce_config_key == 'JOB_ROUTER_SCHEDD2_POOL':
condor_collector_port = (utilities.get_condor_config_val('COLLECTOR_PORT', quiet_undefined=True)
or '9618')
condor_config_value = self._add_port_if_necessary(condor_config_value, condor_collector_port)
if not condor_ce_config_value or condor_ce_config_value != condor_config_value:
condor_ce_config[condor_ce_config_key] = condor_config_value
if condor_ce_config:
buf = utilities.read_file(JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE,
default="# This file is managed by osg-configure\n")
for key, value in condor_ce_config.items():
buf = utilities.add_or_replace_setting(buf, key, value, quote_value=False)
if not utilities.atomic_write(JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, buf):
return False
return True
def setup_gram_config(self):
"""
Populate the gram config file with correct values
Returns True if successful, False otherwise
"""
buf = open(SGEConfiguration.GRAM_CONFIG_FILE).read()
for binfile in ['qsub', 'qstat', 'qdel', 'qconf']:
bin_location = os.path.join(self.options['sge_bin_location'].value, binfile)
if validation.valid_file(bin_location):
buf = utilities.add_or_replace_setting(buf, binfile, bin_location)
for setting in ['sge_cell', 'sge_root', 'sge_config']:
buf = utilities.add_or_replace_setting(buf, setting, self.options[setting].value)
if self.options['seg_enabled'].value:
buf = utilities.add_or_replace_setting(buf, 'log_path', self.options['log_file'].value)
if self.options['default_queue'].value != '':
buf = utilities.add_or_replace_setting(buf, 'default_queue', self.options['default_queue'].value)
if self.options['validate_queues'].value:
buf = utilities.add_or_replace_setting(buf, 'validate_queues', 'yes', quote_value=False)
else:
buf = utilities.add_or_replace_setting(buf, 'validate_queues', 'no', quote_value=False)
if self.options['available_queues'].value != '':
buf = utilities.add_or_replace_setting(buf, 'available_queues', self.options['available_queues'].value)
if not utilities.atomic_write(SGEConfiguration.GRAM_CONFIG_FILE, buf):
return False
return True
def set_pbs_pro_in_blah_config(self):
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
new_value = "yes" if self.opt_val('pbs_flavor') == "pro" else "no"
contents = utilities.add_or_replace_setting(contents,
"pbs_pro",
new_value,
quote_value=False)
utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_blah_disable_wn_proxy_renewal_to_blah_config(self):
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
contents = utilities.add_or_replace_setting(
contents,
"blah_disable_wn_proxy_renewal",
"yes",
quote_value=True)
utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_blah_disable_wn_proxy_renewal_to_blah_config(self):
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
for option, value in [("blah_disable_wn_proxy_renewal", "yes"),
("blah_delegate_renewed_proxies", "no"),
("blah_disable_limited_proxy", "yes")]:
contents = utilities.add_or_replace_setting(contents, option, value,
quote_value=True)
utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_lsf_confpath_to_blah_config(self):
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
contents = utilities.add_or_replace_setting(
contents,
'lsf_confpath',
self.options['lsf_conf'].value,
quote_value=True)
utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_gridmap_to_htcondor_ce_config(self):
contents = utilities.read_file(HTCONDOR_CE_CONFIG_FILE,
default="# This file is managed by osg-configure\n")
if self.options['authorization_method'].value == 'xacml':
# Remove GRIDMAP setting
contents = re.sub(r'(?m)^\s*GRIDMAP\s*=.*?$[\n]?', "", contents)
else:
contents = utilities.add_or_replace_setting(contents, "GRIDMAP", "/etc/grid-security/grid-mapfile",
quote_value=False)
utilities.atomic_write(HTCONDOR_CE_CONFIG_FILE, contents)
def setup_gram_config(self):
"""
Populate the gram config file with correct values
Returns True if successful, False otherwise
"""
buf = open(CondorConfiguration.GRAM_CONFIG_FILE).read()
for binfile in ['condor_submit', 'condor_rm']:
bin_location = os.path.join(self.condor_bin_location, binfile)
if validation.valid_file(bin_location):
buf = utilities.add_or_replace_setting(buf, binfile, bin_location)
if not utilities.blank(self.options['condor_config'].value):
buf = utilities.add_or_replace_setting(buf, 'condor_config', self.options['condor_config'].value)
if not utilities.atomic_write(CondorConfiguration.GRAM_CONFIG_FILE, buf):
return False
return True
def write_htcondor_ce_sentinel(self):
if self.htcondor_gateway_enabled and utilities.ce_installed():
contents = utilities.read_file(
self.HTCONDOR_CE_CONFIG_FILE,
default="# This file is managed by osg-configure\n")
contents = utilities.add_or_replace_setting(contents,
"OSG_CONFIGURED",
"true",
quote_value=False)
utilities.atomic_write(self.HTCONDOR_CE_CONFIG_FILE, contents)
def setup_blah_config(self):
"""
Populate blah.config with correct values
Return True if successful, False otherwise
"""
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
contents = utilities.add_or_replace_setting(
contents,
"sge_rootpath",
self.options['sge_root'].value,
quote_value=True)
contents = utilities.add_or_replace_setting(
contents,
"sge_cellname",
self.options['sge_cell'].value,
quote_value=True)
return utilities.atomic_write(self.BLAH_CONFIG, contents)
return False
def setup_gram_config(self):
"""
Populate the gram config file with correct values
Returns True if successful, False otherwise
"""
buf = open(CondorConfiguration.GRAM_CONFIG_FILE).read()
for binfile in ['condor_submit', 'condor_rm']:
bin_location = os.path.join(self.condor_bin_location, binfile)
if validation.valid_file(bin_location):
buf = utilities.add_or_replace_setting(buf, binfile,
bin_location)
if not utilities.blank(self.options['condor_config'].value):
buf = utilities.add_or_replace_setting(
buf, 'condor_config', self.options['condor_config'].value)
if not utilities.atomic_write(CondorConfiguration.GRAM_CONFIG_FILE,
buf):
return False
return True
def setup_gram_config(self):
"""
Populate the gram config file with correct values
Returns True if successful, False otherwise
"""
contents = open(SlurmConfiguration.GRAM_CONFIG_FILE).read()
for binfile in ['qsub', 'qstat', 'qdel']:
bin_location = os.path.join(self.slurm_bin_location, binfile)
if validation.valid_file(bin_location):
contents = utilities.add_or_replace_setting(contents, binfile, bin_location)
if not utilities.atomic_write(SlurmConfiguration.GRAM_CONFIG_FILE, contents):
return False
return True
def write_binpaths_to_blah_config(self, jobmanager, submit_binpath):
"""
Change the *_binpath variables in /etc/blah.config for the given
jobmanager to point to the locations specified by the user in the
config for that jobmanager. Does not do anything if /etc/blah.config
is missing (e.g. if blahp is not installed).
:param jobmanager: The name of a job manager that has a _binpath
variable in /etc/blah.config
:param submit_binpath: The fully-qualified path to the submit
executables for that jobmanager
"""
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
contents = utilities.add_or_replace_setting(contents, jobmanager + "_binpath", submit_binpath,
quote_value=True)
utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_binpaths_to_blah_config(self, jobmanager, submit_binpath):
"""
Change the *_binpath variables in /etc/blah.config for the given
jobmanager to point to the locations specified by the user in the
config for that jobmanager. Does not do anything if /etc/blah.config
is missing (e.g. if blahp is not installed).
:param jobmanager: The name of a job manager that has a _binpath
variable in /etc/blah.config
:param submit_binpath: The fully-qualified path to the submit
executables for that jobmanager
"""
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
contents = utilities.add_or_replace_setting(contents, jobmanager + "_binpath", submit_binpath,
quote_value=False)
utilities.atomic_write(self.BLAH_CONFIG, contents)
def setup_gram_config(self):
"""
Populate the gram config file with correct values
Returns True if successful, False otherwise
"""
contents = open(SlurmConfiguration.GRAM_CONFIG_FILE).read()
for binfile in ['qsub', 'qstat', 'qdel']:
bin_location = os.path.join(self.slurm_bin_location, binfile)
if validation.valid_file(bin_location):
contents = utilities.add_or_replace_setting(
contents, binfile, bin_location)
if not utilities.atomic_write(SlurmConfiguration.GRAM_CONFIG_FILE,
contents):
return False
return True
def setup_gram_config(self):
"""
Populate the gram config file with correct values
Returns True if successful, False otherwise
"""
buf = open(SGEConfiguration.GRAM_CONFIG_FILE).read()
for binfile in ['qsub', 'qstat', 'qdel', 'qconf']:
bin_location = os.path.join(self.options['sge_bin_location'].value,
binfile)
if validation.valid_file(bin_location):
buf = utilities.add_or_replace_setting(buf, binfile,
bin_location)
for setting in ['sge_cell', 'sge_root', 'sge_config']:
buf = utilities.add_or_replace_setting(buf, setting,
self.options[setting].value)
if self.options['seg_enabled'].value:
buf = utilities.add_or_replace_setting(
buf, 'log_path', self.options['log_file'].value)
if self.options['default_queue'].value != '':
buf = utilities.add_or_replace_setting(
buf, 'default_queue', self.options['default_queue'].value)
if self.options['validate_queues'].value:
buf = utilities.add_or_replace_setting(buf,
'validate_queues',
'yes',
quote_value=False)
else:
buf = utilities.add_or_replace_setting(buf,
'validate_queues',
'no',
quote_value=False)
if self.options['available_queues'].value != '':
buf = utilities.add_or_replace_setting(
buf, 'available_queues',
self.options['available_queues'].value)
if not utilities.atomic_write(SGEConfiguration.GRAM_CONFIG_FILE, buf):
return False
return True
def write_lsf_confpath_to_blah_config(self):
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
contents = utilities.add_or_replace_setting(contents, 'lsf_confpath', self.options['lsf_conf'].value,
quote_value=True)
utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_blah_disable_wn_proxy_renewal_to_blah_config(self):
if os.path.exists(self.BLAH_CONFIG):
contents = utilities.read_file(self.BLAH_CONFIG)
contents = utilities.add_or_replace_setting(contents, "blah_disable_wn_proxy_renewal", "yes",
quote_value=False)
utilities.atomic_write(self.BLAH_CONFIG, contents)
def setup_htcondor_ce_config(self):
"""
Populate the config file that tells htcondor-ce where the condor
pool is and where the spool directory is.
Returns True if successful, False otherwise
"""
if not utilities.rpm_installed('htcondor-ce'):
self.log(
"Unable to configure htcondor-ce for Condor: htcondor-ce not installed",
level=logging.ERROR)
return False
def get_condor_ce_config_val(variable):
return utilities.get_condor_config_val(
variable,
executable='condor_ce_config_val',
quiet_undefined=True)
# Get values for the settings we want to update. We can get the
# values from condor_config_val; in the case of JOB_ROUTER_SCHEDD2_NAME,
# we have FULL_HOSTNAME as a fallback in case SCHEDD_NAME is missing.
# We also get the current / default value from condor_ce_config_val;
# only update the setting in case the value from
# condor_config_val is different from the value from condor_ce_config_val.
condor_ce_config = {}
for condor_ce_config_key, condor_config_keys in [
('JOB_ROUTER_SCHEDD2_NAME', ['SCHEDD_NAME', 'FULL_HOSTNAME']),
('JOB_ROUTER_SCHEDD2_POOL', ['COLLECTOR_HOST']),
('JOB_ROUTER_SCHEDD2_SPOOL', ['SPOOL'])
]:
condor_config_value = None
for condor_config_value in (utilities.get_condor_config_val(
k, quiet_undefined=True) for k in condor_config_keys):
if condor_config_value:
break
condor_ce_config_value = get_condor_ce_config_val(
condor_ce_config_key)
if not (condor_config_value or condor_ce_config_value):
self.log(
"Unable to determine value for %s from %s and default not set; check your Condor config"
% (condor_ce_config_key, ' or '.join(condor_config_keys)),
level=logging.ERROR)
return False
elif not condor_config_value:
continue # can't set anything for this
# Special case for JOB_ROUTER_SCHEDD2_POOL: append port if necessary (SOFTWARE-1744)
if condor_ce_config_key == 'JOB_ROUTER_SCHEDD2_POOL':
condor_collector_port = (utilities.get_condor_config_val(
'COLLECTOR_PORT', quiet_undefined=True) or '9618')
condor_config_value = self._add_port_if_necessary(
condor_config_value, condor_collector_port)
if not condor_ce_config_value or condor_ce_config_value != condor_config_value:
condor_ce_config[condor_ce_config_key] = condor_config_value
if condor_ce_config:
buf = utilities.read_file(
JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE,
default="# This file is managed by osg-configure\n")
for key, value in condor_ce_config.items():
buf = utilities.add_or_replace_setting(buf,
key,
value,
quote_value=False)
if not utilities.atomic_write(
JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, buf):
return False
return True
def write_htcondor_ce_sentinel(self):
if self.htcondor_gateway_enabled:
contents = utilities.read_file(self.HTCONDOR_CE_CONFIG_FILE,
default="# This file is managed by osg-configure\n")
contents = utilities.add_or_replace_setting(contents, "OSG_CONFIGURED", "true", quote_value=False)
utilities.atomic_write(self.HTCONDOR_CE_CONFIG_FILE, contents)
