def setup_gram_config(self): """ Populate the gram config file with correct values Returns True if successful, False otherwise """ buf = open(LSFConfiguration.GRAM_CONFIG_FILE).read() for binfile in ['bsub', 'bqueues', 'bjobs', 'bhist', 'bacct', 'bkill']: bin_location = os.path.join(self.lsf_bin_location, binfile) if validation.valid_file(bin_location): buf = utilities.add_or_replace_setting(buf, binfile, 'bin_location') if self.options['seg_enabled'].value: if (self.options['log_directory'].value is None or not validation.valid_directory(self.options['log_directory'].value)): mesg = "%s is not a valid directory location " % self.options['log_directory'].value mesg += "for lsf log files" self.log(mesg, section=self.config_section, option='log_directory', level=logging.ERROR) return False buf = utilities.add_or_replace_setting(buf, 'log_path', self.options['log_directory'].value) buf = utilities.add_or_replace_setting(buf, 'lsf_profile', self.options['lsf_profile'].value) if not utilities.atomic_write(LSFConfiguration.GRAM_CONFIG_FILE, buf): return False return True
def setup_gram_config(self): """ Populate the gram config file with correct values Returns True if successful, False otherwise """ buf = open(LSFConfiguration.GRAM_CONFIG_FILE).read() for binfile in ['bsub', 'bqueues', 'bjobs', 'bhist', 'bacct', 'bkill']: bin_location = os.path.join(self.lsf_bin_location, binfile) if validation.valid_file(bin_location): buf = utilities.add_or_replace_setting(buf, binfile, 'bin_location') if self.options['seg_enabled'].value: if (self.options['log_directory'].value is None or not validation.valid_directory( self.options['log_directory'].value)): mesg = "%s is not a valid directory location " % self.options[ 'log_directory'].value mesg += "for lsf log files" self.log(mesg, section=self.config_section, option='log_directory', level=logging.ERROR) return False buf = utilities.add_or_replace_setting( buf, 'log_path', self.options['log_directory'].value) buf = utilities.add_or_replace_setting( buf, 'lsf_profile', self.options['lsf_profile'].value) if not utilities.atomic_write(LSFConfiguration.GRAM_CONFIG_FILE, buf): return False return True
def setup_gram_config(self): """ Populate the gram config file with correct values Returns True if successful, False otherwise """ contents = open(PBSConfiguration.GRAM_CONFIG_FILE).read() for binfile in ['qsub', 'qstat', 'qdel']: bin_location = os.path.join(self.pbs_bin_location, binfile) if validation.valid_file(bin_location): contents = utilities.add_or_replace_setting(contents, binfile, bin_location) if self.options['pbs_server'].value != '': contents = utilities.add_or_replace_setting(contents, 'pbs_default', self.options['pbs_server'].value) if self.options['seg_enabled'].value: if (self.options['log_directory'].value is None or not validation.valid_directory(self.options['log_directory'].value)): mesg = "%s is not a valid directory location " % self.options['log_directory'].value mesg += "for pbs log files" self.log(mesg, section=self.config_section, option='log_directory', level=logging.ERROR) return False contents = utilities.add_or_replace_setting(contents, 'log_path', self.options['log_directory'].value) if not utilities.atomic_write(PBSConfiguration.GRAM_CONFIG_FILE, contents): return False return True
def _write_route_config_vars(self): """ Write condor-ce config attributes for the bosco job route. Sets values for: - BOSCO_RMS - BOSCO_ENDPOINT """ contents = utilities.read_file(self.HTCONDOR_CE_CONFIG_FILE, default="# This file is managed by osg-configure\n") contents = utilities.add_or_replace_setting(contents, "BOSCO_RMS", self.options['batch'].value, quote_value=False) contents = utilities.add_or_replace_setting(contents, "BOSCO_ENDPOINT", self.options['endpoint'].value, quote_value=False) utilities.atomic_write(self.HTCONDOR_CE_CONFIG_FILE, contents)
def setup_blah_config(self): """ Populate blah.config with correct values Return True if successful, False otherwise """ if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) contents = utilities.add_or_replace_setting(contents, "sge_rootpath", self.options['sge_root'].value, quote_value=True) contents = utilities.add_or_replace_setting(contents, "sge_cellname", self.options['sge_cell'].value, quote_value=True) return utilities.atomic_write(self.BLAH_CONFIG, contents) return False
def setup_htcondor_ce_config(self): """ Populate the config file that tells htcondor-ce where the condor pool is and where the spool directory is. Returns True if successful, False otherwise """ if not utilities.rpm_installed('htcondor-ce'): self.log("Unable to configure htcondor-ce for Condor: htcondor-ce not installed", level=logging.ERROR) return False def get_condor_ce_config_val(variable): return utilities.get_condor_config_val(variable, executable='condor_ce_config_val', quiet_undefined=True) # Get values for the settings we want to update. We can get the # values from condor_config_val; in the case of JOB_ROUTER_SCHEDD2_NAME, # we have FULL_HOSTNAME as a fallback in case SCHEDD_NAME is missing. # We also get the current / default value from condor_ce_config_val; # only update the setting in case the value from # condor_config_val is different from the value from condor_ce_config_val. condor_ce_config = {} for condor_ce_config_key, condor_config_keys in [ ('JOB_ROUTER_SCHEDD2_NAME', ['SCHEDD_NAME', 'FULL_HOSTNAME']), ('JOB_ROUTER_SCHEDD2_POOL', ['COLLECTOR_HOST']), ('JOB_ROUTER_SCHEDD2_SPOOL', ['SPOOL'])]: condor_config_value = None for condor_config_value in (utilities.get_condor_config_val(k, quiet_undefined=True) for k in condor_config_keys): if condor_config_value: break condor_ce_config_value = get_condor_ce_config_val(condor_ce_config_key) if not (condor_config_value or condor_ce_config_value): self.log("Unable to determine value for %s from %s and default not set; check your Condor config" % (condor_ce_config_key, ' or '.join(condor_config_keys)), level=logging.ERROR) return False elif not condor_config_value: continue # can't set anything for this # Special case for JOB_ROUTER_SCHEDD2_POOL: append port if necessary (SOFTWARE-1744) if condor_ce_config_key == 'JOB_ROUTER_SCHEDD2_POOL': condor_collector_port = (utilities.get_condor_config_val('COLLECTOR_PORT', quiet_undefined=True) or '9618') condor_config_value = self._add_port_if_necessary(condor_config_value, condor_collector_port) if not condor_ce_config_value or condor_ce_config_value != condor_config_value: condor_ce_config[condor_ce_config_key] = condor_config_value if condor_ce_config: buf = utilities.read_file(JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, default="# This file is managed by osg-configure\n") for key, value in condor_ce_config.items(): buf = utilities.add_or_replace_setting(buf, key, value, quote_value=False) if not utilities.atomic_write(JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, buf): return False return True
def setup_gram_config(self): """ Populate the gram config file with correct values Returns True if successful, False otherwise """ buf = open(SGEConfiguration.GRAM_CONFIG_FILE).read() for binfile in ['qsub', 'qstat', 'qdel', 'qconf']: bin_location = os.path.join(self.options['sge_bin_location'].value, binfile) if validation.valid_file(bin_location): buf = utilities.add_or_replace_setting(buf, binfile, bin_location) for setting in ['sge_cell', 'sge_root', 'sge_config']: buf = utilities.add_or_replace_setting(buf, setting, self.options[setting].value) if self.options['seg_enabled'].value: buf = utilities.add_or_replace_setting(buf, 'log_path', self.options['log_file'].value) if self.options['default_queue'].value != '': buf = utilities.add_or_replace_setting(buf, 'default_queue', self.options['default_queue'].value) if self.options['validate_queues'].value: buf = utilities.add_or_replace_setting(buf, 'validate_queues', 'yes', quote_value=False) else: buf = utilities.add_or_replace_setting(buf, 'validate_queues', 'no', quote_value=False) if self.options['available_queues'].value != '': buf = utilities.add_or_replace_setting(buf, 'available_queues', self.options['available_queues'].value) if not utilities.atomic_write(SGEConfiguration.GRAM_CONFIG_FILE, buf): return False return True
def set_pbs_pro_in_blah_config(self): if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) new_value = "yes" if self.opt_val('pbs_flavor') == "pro" else "no" contents = utilities.add_or_replace_setting(contents, "pbs_pro", new_value, quote_value=False) utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_blah_disable_wn_proxy_renewal_to_blah_config(self): if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) contents = utilities.add_or_replace_setting( contents, "blah_disable_wn_proxy_renewal", "yes", quote_value=True) utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_blah_disable_wn_proxy_renewal_to_blah_config(self): if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) for option, value in [("blah_disable_wn_proxy_renewal", "yes"), ("blah_delegate_renewed_proxies", "no"), ("blah_disable_limited_proxy", "yes")]: contents = utilities.add_or_replace_setting(contents, option, value, quote_value=True) utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_lsf_confpath_to_blah_config(self): if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) contents = utilities.add_or_replace_setting( contents, 'lsf_confpath', self.options['lsf_conf'].value, quote_value=True) utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_gridmap_to_htcondor_ce_config(self): contents = utilities.read_file(HTCONDOR_CE_CONFIG_FILE, default="# This file is managed by osg-configure\n") if self.options['authorization_method'].value == 'xacml': # Remove GRIDMAP setting contents = re.sub(r'(?m)^\s*GRIDMAP\s*=.*?$[\n]?', "", contents) else: contents = utilities.add_or_replace_setting(contents, "GRIDMAP", "/etc/grid-security/grid-mapfile", quote_value=False) utilities.atomic_write(HTCONDOR_CE_CONFIG_FILE, contents)
def setup_gram_config(self): """ Populate the gram config file with correct values Returns True if successful, False otherwise """ buf = open(CondorConfiguration.GRAM_CONFIG_FILE).read() for binfile in ['condor_submit', 'condor_rm']: bin_location = os.path.join(self.condor_bin_location, binfile) if validation.valid_file(bin_location): buf = utilities.add_or_replace_setting(buf, binfile, bin_location) if not utilities.blank(self.options['condor_config'].value): buf = utilities.add_or_replace_setting(buf, 'condor_config', self.options['condor_config'].value) if not utilities.atomic_write(CondorConfiguration.GRAM_CONFIG_FILE, buf): return False return True
def write_htcondor_ce_sentinel(self): if self.htcondor_gateway_enabled and utilities.ce_installed(): contents = utilities.read_file( self.HTCONDOR_CE_CONFIG_FILE, default="# This file is managed by osg-configure\n") contents = utilities.add_or_replace_setting(contents, "OSG_CONFIGURED", "true", quote_value=False) utilities.atomic_write(self.HTCONDOR_CE_CONFIG_FILE, contents)
def setup_blah_config(self): """ Populate blah.config with correct values Return True if successful, False otherwise """ if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) contents = utilities.add_or_replace_setting( contents, "sge_rootpath", self.options['sge_root'].value, quote_value=True) contents = utilities.add_or_replace_setting( contents, "sge_cellname", self.options['sge_cell'].value, quote_value=True) return utilities.atomic_write(self.BLAH_CONFIG, contents) return False
def setup_gram_config(self): """ Populate the gram config file with correct values Returns True if successful, False otherwise """ buf = open(CondorConfiguration.GRAM_CONFIG_FILE).read() for binfile in ['condor_submit', 'condor_rm']: bin_location = os.path.join(self.condor_bin_location, binfile) if validation.valid_file(bin_location): buf = utilities.add_or_replace_setting(buf, binfile, bin_location) if not utilities.blank(self.options['condor_config'].value): buf = utilities.add_or_replace_setting( buf, 'condor_config', self.options['condor_config'].value) if not utilities.atomic_write(CondorConfiguration.GRAM_CONFIG_FILE, buf): return False return True
def setup_gram_config(self): """ Populate the gram config file with correct values Returns True if successful, False otherwise """ contents = open(SlurmConfiguration.GRAM_CONFIG_FILE).read() for binfile in ['qsub', 'qstat', 'qdel']: bin_location = os.path.join(self.slurm_bin_location, binfile) if validation.valid_file(bin_location): contents = utilities.add_or_replace_setting(contents, binfile, bin_location) if not utilities.atomic_write(SlurmConfiguration.GRAM_CONFIG_FILE, contents): return False return True
def write_binpaths_to_blah_config(self, jobmanager, submit_binpath): """ Change the *_binpath variables in /etc/blah.config for the given jobmanager to point to the locations specified by the user in the config for that jobmanager. Does not do anything if /etc/blah.config is missing (e.g. if blahp is not installed). :param jobmanager: The name of a job manager that has a _binpath variable in /etc/blah.config :param submit_binpath: The fully-qualified path to the submit executables for that jobmanager """ if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) contents = utilities.add_or_replace_setting(contents, jobmanager + "_binpath", submit_binpath, quote_value=True) utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_binpaths_to_blah_config(self, jobmanager, submit_binpath): """ Change the *_binpath variables in /etc/blah.config for the given jobmanager to point to the locations specified by the user in the config for that jobmanager. Does not do anything if /etc/blah.config is missing (e.g. if blahp is not installed). :param jobmanager: The name of a job manager that has a _binpath variable in /etc/blah.config :param submit_binpath: The fully-qualified path to the submit executables for that jobmanager """ if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) contents = utilities.add_or_replace_setting(contents, jobmanager + "_binpath", submit_binpath, quote_value=False) utilities.atomic_write(self.BLAH_CONFIG, contents)
def setup_gram_config(self): """ Populate the gram config file with correct values Returns True if successful, False otherwise """ contents = open(SlurmConfiguration.GRAM_CONFIG_FILE).read() for binfile in ['qsub', 'qstat', 'qdel']: bin_location = os.path.join(self.slurm_bin_location, binfile) if validation.valid_file(bin_location): contents = utilities.add_or_replace_setting( contents, binfile, bin_location) if not utilities.atomic_write(SlurmConfiguration.GRAM_CONFIG_FILE, contents): return False return True
def setup_gram_config(self): """ Populate the gram config file with correct values Returns True if successful, False otherwise """ buf = open(SGEConfiguration.GRAM_CONFIG_FILE).read() for binfile in ['qsub', 'qstat', 'qdel', 'qconf']: bin_location = os.path.join(self.options['sge_bin_location'].value, binfile) if validation.valid_file(bin_location): buf = utilities.add_or_replace_setting(buf, binfile, bin_location) for setting in ['sge_cell', 'sge_root', 'sge_config']: buf = utilities.add_or_replace_setting(buf, setting, self.options[setting].value) if self.options['seg_enabled'].value: buf = utilities.add_or_replace_setting( buf, 'log_path', self.options['log_file'].value) if self.options['default_queue'].value != '': buf = utilities.add_or_replace_setting( buf, 'default_queue', self.options['default_queue'].value) if self.options['validate_queues'].value: buf = utilities.add_or_replace_setting(buf, 'validate_queues', 'yes', quote_value=False) else: buf = utilities.add_or_replace_setting(buf, 'validate_queues', 'no', quote_value=False) if self.options['available_queues'].value != '': buf = utilities.add_or_replace_setting( buf, 'available_queues', self.options['available_queues'].value) if not utilities.atomic_write(SGEConfiguration.GRAM_CONFIG_FILE, buf): return False return True
def write_lsf_confpath_to_blah_config(self): if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) contents = utilities.add_or_replace_setting(contents, 'lsf_confpath', self.options['lsf_conf'].value, quote_value=True) utilities.atomic_write(self.BLAH_CONFIG, contents)
def write_blah_disable_wn_proxy_renewal_to_blah_config(self): if os.path.exists(self.BLAH_CONFIG): contents = utilities.read_file(self.BLAH_CONFIG) contents = utilities.add_or_replace_setting(contents, "blah_disable_wn_proxy_renewal", "yes", quote_value=False) utilities.atomic_write(self.BLAH_CONFIG, contents)
def setup_htcondor_ce_config(self): """ Populate the config file that tells htcondor-ce where the condor pool is and where the spool directory is. Returns True if successful, False otherwise """ if not utilities.rpm_installed('htcondor-ce'): self.log( "Unable to configure htcondor-ce for Condor: htcondor-ce not installed", level=logging.ERROR) return False def get_condor_ce_config_val(variable): return utilities.get_condor_config_val( variable, executable='condor_ce_config_val', quiet_undefined=True) # Get values for the settings we want to update. We can get the # values from condor_config_val; in the case of JOB_ROUTER_SCHEDD2_NAME, # we have FULL_HOSTNAME as a fallback in case SCHEDD_NAME is missing. # We also get the current / default value from condor_ce_config_val; # only update the setting in case the value from # condor_config_val is different from the value from condor_ce_config_val. condor_ce_config = {} for condor_ce_config_key, condor_config_keys in [ ('JOB_ROUTER_SCHEDD2_NAME', ['SCHEDD_NAME', 'FULL_HOSTNAME']), ('JOB_ROUTER_SCHEDD2_POOL', ['COLLECTOR_HOST']), ('JOB_ROUTER_SCHEDD2_SPOOL', ['SPOOL']) ]: condor_config_value = None for condor_config_value in (utilities.get_condor_config_val( k, quiet_undefined=True) for k in condor_config_keys): if condor_config_value: break condor_ce_config_value = get_condor_ce_config_val( condor_ce_config_key) if not (condor_config_value or condor_ce_config_value): self.log( "Unable to determine value for %s from %s and default not set; check your Condor config" % (condor_ce_config_key, ' or '.join(condor_config_keys)), level=logging.ERROR) return False elif not condor_config_value: continue # can't set anything for this # Special case for JOB_ROUTER_SCHEDD2_POOL: append port if necessary (SOFTWARE-1744) if condor_ce_config_key == 'JOB_ROUTER_SCHEDD2_POOL': condor_collector_port = (utilities.get_condor_config_val( 'COLLECTOR_PORT', quiet_undefined=True) or '9618') condor_config_value = self._add_port_if_necessary( condor_config_value, condor_collector_port) if not condor_ce_config_value or condor_ce_config_value != condor_config_value: condor_ce_config[condor_ce_config_key] = condor_config_value if condor_ce_config: buf = utilities.read_file( JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, default="# This file is managed by osg-configure\n") for key, value in condor_ce_config.items(): buf = utilities.add_or_replace_setting(buf, key, value, quote_value=False) if not utilities.atomic_write( JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, buf): return False return True
def write_htcondor_ce_sentinel(self): if self.htcondor_gateway_enabled: contents = utilities.read_file(self.HTCONDOR_CE_CONFIG_FILE, default="# This file is managed by osg-configure\n") contents = utilities.add_or_replace_setting(contents, "OSG_CONFIGURED", "true", quote_value=False) utilities.atomic_write(self.HTCONDOR_CE_CONFIG_FILE, contents)