def get(self, attribute=None):
"""GET handler"""
parser = reqparse.RequestParser(trim=True)
parser.add_argument("daemon", choices=list(self.DAEMON_TYPES_MAP.keys()))
args = parser.parse_args()
if args.daemon:
daemon_ad = Collector().locate(self.DAEMON_TYPES_MAP[args.daemon])
param = RemoteParam(daemon_ad)
else:
htcondor.reload_config()
param = htcondor.param
param_lower = utils.deep_lcasekeys(param)
if attribute:
if not validate_attribute(attribute):
abort(400, message="Invalid attribute")
if attribute:
try:
return param_lower[attribute.lower()]
except KeyError as err:
abort(404, message=str(err))
return param_lower
def launch_daemons(self, daemons=["MASTER", "COLLECTOR"], config={}):
makedirs_ignore_exist(htcondor.param["LOG"])
makedirs_ignore_exist(htcondor.param["LOCK"])
makedirs_ignore_exist(htcondor.param["EXECUTE"])
makedirs_ignore_exist(htcondor.param["SPOOL"])
makedirs_ignore_exist(htcondor.param["RUN"])
remove_ignore_missing(htcondor.param["MASTER_ADDRESS_FILE"])
remove_ignore_missing(htcondor.param["COLLECTOR_ADDRESS_FILE"])
remove_ignore_missing(htcondor.param["SCHEDD_ADDRESS_FILE"])
for key, val in config.items():
os.environ["_condor_%s" % key] = val
if "COLLECTOR" in daemons:
os.environ["_condor_PORT"] = "9622"
os.environ["_condor_COLLECTOR_ARGS"] = "-port $(PORT)"
os.environ["_condor_COLLECTOR_HOST"] = "$(CONDOR_HOST):$(PORT)"
if 'MASTER' not in daemons:
daemons.append('MASTER')
os.environ["_condor_DAEMON_LIST"] = ", ".join(daemons)
htcondor.reload_config()
self.pid = os.fork()
if not self.pid:
try:
try:
os.execvp("condor_master", ["condor_master", "-f"])
except:
e = sys.exc_info()[1]
print(str(e))
finally:
os._exit(1)
global master_pid
master_pid = self.pid
for daemon in daemons:
self.waitLocalDaemon(daemon)
def SetupLocalEnvironment(self):
Utils.MakedirsIgnoreExist(self._local_dir)
Utils.MakedirsIgnoreExist(self._execute_path)
Utils.MakedirsIgnoreExist(self._log_path)
Utils.MakedirsIgnoreExist(self._lock_path)
Utils.MakedirsIgnoreExist(self._run_path)
Utils.MakedirsIgnoreExist(self._spool_path)
Utils.RemoveIgnoreMissing(htcondor.param["MASTER_ADDRESS_FILE"])
Utils.RemoveIgnoreMissing(htcondor.param["COLLECTOR_ADDRESS_FILE"])
Utils.RemoveIgnoreMissing(htcondor.param["SCHEDD_ADDRESS_FILE"])
# Create a config file in this test's local directory based on existing
# config settings
os.system("condor_config_val -write:up " + self._local_config)
# Add whatever internal config values we need
config = "LOCAL_DIR = " + self._local_path + "\n"
config += "EXECUTE = " + self._execute_path + "\n"
config += "LOCK = " + self._lock_path + "\n"
config += "RUN = " + self._run_path + "\n"
config += "SPOOL = " + self._spool_path + "\n"
config += "COLLECTOR_HOST = $(CONDOR_HOST):0\n"
config_file = open(self._local_config, "a")
config_file.write(config)
config_file.close()
self.SetCondorConfig()
# MRC: What does this function actually do?
htcondor.reload_config()
def CondorCluster(self, job_args):
if self._schedd is None:
try:
original = self.SetCondorConfig()
htcondor.reload_config()
name = htcondor.param["SCHEDD_ADDRESS_FILE"]
f = open(name, 'r')
contents = f.read()
f.close()
c = classad.ClassAd()
(address, version, platform) = contents.splitlines()
c["MyAddress"] = address
c["Name"] = "Unknown"
c["CondorVersion"] = version
# Utils.TLog( "[PC: {0}] Constructing schedd from address '{1}' with version '{2}'".format(self._name, address, version))
self._schedd = htcondor.Schedd(c)
except IOError as ioe:
# Utils.TLog( "[PC: {0}] Constructing default schedd because of IOError {1}".format(self._name, str(ioe)))
self._schedd = htcondor.Schedd()
finally:
os.environ["CONDOR_CONFIG"] = original
htcondor.reload_config()
return CondorCluster(job_args, self._schedd)
def fetch(self, constraint=None, format_list=None, condor_config=None):
"""
Fetch resource classads and return a list of evaluated classads
"""
if self.logger is not None:
self.logger.debug("in CondorStatus fetch")
results = []
constraint = bindings_friendly_constraint(constraint)
attrs = bindings_friendly_attrs(format_list)
adtype = resource_str_to_py_adtype(self.resource_str)
old_condor_config_env = os.environ.get("CONDOR_CONFIG")
try:
if condor_config and os.path.exists(condor_config):
os.environ["CONDOR_CONFIG"] = condor_config
htcondor.reload_config()
if self.pool_name:
collector = htcondor.Collector(str(self.pool_name))
else:
collector = htcondor.Collector()
classads = collector.query(adtype, constraint, attrs)
results = eval_classad_expr(classads, format_list=format_list)
except Exception as ex:
p = "default"
if self.pool_name is not None:
p = self.pool_name
err_str = f"Error querying pool {p} using python bindings: {ex}"
raise QueryError(err_str).with_traceback(sys.exc_info()[2])
finally:
if old_condor_config_env:
os.environ["CONDOR_CONFIG"] = old_condor_config_env
return results
def unset(self):
"""Un-set ``CONDOR_CONFIG`` and tell HTCondor to reconfigure."""
if self.previous_value is not None:
_set_env_var("CONDOR_CONFIG", self.previous_value)
htcondor.reload_config()
else:
_unset_env_var("CONDOR_CONFIG")
def launch_daemons(self, daemons=["MASTER", "COLLECTOR"]):
makedirs_ignore_exist(htcondor.param["LOG"])
makedirs_ignore_exist(htcondor.param["LOCK"])
makedirs_ignore_exist(htcondor.param["EXECUTE"])
makedirs_ignore_exist(htcondor.param["SPOOL"])
makedirs_ignore_exist(htcondor.param["RUN"])
remove_ignore_missing(htcondor.param["MASTER_ADDRESS_FILE"])
remove_ignore_missing(htcondor.param["COLLECTOR_ADDRESS_FILE"])
remove_ignore_missing(htcondor.param["SCHEDD_ADDRESS_FILE"])
if "COLLECTOR" in daemons:
os.environ["_condor_PORT"] = "9622"
os.environ["_condor_COLLECTOR_ARGS"] = "-port $(PORT)"
os.environ["_condor_COLLECTOR_HOST"] = "$(CONDOR_HOST):$(PORT)"
if 'MASTER' not in daemons:
daemons.append('MASTER')
os.environ["_condor_DAEMON_LIST"] = ", ".join(daemons)
htcondor.reload_config()
self.pid = os.fork()
if not self.pid:
try:
try:
os.execvp("condor_master", ["condor_master", "-f"])
except Exception, e:
print str(e)
finally:
os._exit(1)
for daemon in daemons:
self.waitLocalDaemon(daemon)
def __enter__(self):
self.previous_value = os.environ.get("CONDOR_CONFIG", None)
set_env_var("CONDOR_CONFIG", self.config_file.as_posix())
htcondor.reload_config()
return self
def get_factory_version(node_name):
htcondor.reload_config()
collector = htcondor.Collector(node_name)
adtype = htcondor.AdTypes.Any
constraint = 'MyType == "glidefactoryglobal"'
results = collector.query(adtype, constraint, ['GlideinWMSVersion'])
return results[0]['GlideinWMSVersion']
def SetupLocalEnvironment(self):
Utils.MakedirsIgnoreExist(self._local_dir)
Utils.MakedirsIgnoreExist(self._execute_path)
Utils.MakedirsIgnoreExist(self._log_path)
Utils.MakedirsIgnoreExist(self._lock_path)
Utils.MakedirsIgnoreExist(self._run_path)
Utils.MakedirsIgnoreExist(self._spool_path)
Utils.RemoveIgnoreMissing(htcondor.param["MASTER_ADDRESS_FILE"])
Utils.RemoveIgnoreMissing(htcondor.param["COLLECTOR_ADDRESS_FILE"])
Utils.RemoveIgnoreMissing(htcondor.param["SCHEDD_ADDRESS_FILE"])
# Create a config file in this test's local directory based on existing
# config settings
os.system("condor_config_val -write:up " + self._local_config)
# Add whatever internal config values we need
config = "LOCAL_DIR = " + self._local_path + "\n"
config += "EXECUTE = " + self._execute_path + "\n"
config += "LOCK = " + self._lock_path + "\n"
config += "RUN = " + self._run_path + "\n"
config += "SPOOL = " + self._spool_path + "\n"
config += "COLLECTOR_HOST = $(CONDOR_HOST):0\n"
config_file = open(self._local_config, "a")
config_file.write(config)
config_file.close()
self.SetCondorConfig()
# MRC: What does this function actually do?
htcondor.reload_config()
def get_factory_version(node_name):
htcondor.reload_config()
collector = htcondor.Collector(node_name)
adtype = htcondor.AdTypes.Any
constraint = 'MyType == "glidefactoryglobal"'
results = collector.query(adtype, constraint, ['GlideinWMSVersion'])
return results[0]['GlideinWMSVersion']
def fetch_using_bindings(self, constraint=None, format_list=None):
"""
Fetch the results using htcondor-python bindings
"""
results_dict = {} # defined here in case of exception
constraint = bindings_friendly_constraint(constraint)
attrs = bindings_friendly_attrs(format_list)
adtype = resource_str_to_py_adtype(self.resource_str)
self.security_obj.save_state()
try:
self.security_obj.enforce_requests()
htcondor.reload_config()
if self.pool_name:
collector = htcondor.Collector(str(self.pool_name))
else:
collector = htcondor.Collector()
results = collector.query(adtype, constraint, attrs)
results_dict = list2dict(results, self.group_attribute)
except Exception as ex:
p = 'default'
if self.pool_name is not None:
p = self.pool_name
err_str = 'Error querying pool %s using python bindings: %s' % (p,
ex)
raise PBError(err_str), None, sys.exc_info()[2]
finally:
self.security_obj.restore_state()
return results_dict
def setUp(self):
self.pid = -1
os.environ["_condor_MASTER"] = os.path.join(os.getcwd(), "../condor_master.V6/condor_master")
os.environ["_condor_COLLECTOR"] = os.path.join(os.getcwd(), "../condor_collector.V6/condor_collector")
os.environ["_condor_SCHEDD"] = os.path.join(os.getcwd(), "../condor_schedd.V6/condor_schedd")
os.environ["_condor_PROCD"] = os.path.join(os.getcwd(), "../condor_procd/condor_procd")
os.environ["_condor_STARTD"] = os.path.join(os.getcwd(), "../condor_startd.V6/condor_startd")
os.environ["_condor_STARTER"] = os.path.join(os.getcwd(), "../condor_starter.V6.1/condor_starter")
os.environ["_condor_NEGOTIATOR"] = os.path.join(os.getcwd(), "../condor_negotiator.V6/condor_negotiator")
os.environ["_condor_SHADOW"] = os.path.join(os.getcwd(), "../condor_shadow.V6.1/condor_shadow")
os.environ["_condor_CONDOR_HOST"] = socket.getfqdn()
os.environ["_condor_LOCAL_DIR"] = testdir
os.environ["_condor_LOG"] = '$(LOCAL_DIR)/log'
os.environ["_condor_LOCK"] = '$(LOCAL_DIR)/lock'
os.environ["_condor_RUN"] = '$(LOCAL_DIR)/run'
os.environ["_condor_COLLECTOR_NAME"] = "python_classad_tests"
os.environ["_condor_SCHEDD_NAME"] = "python_classad_tests"
os.environ["_condor_MASTER_ADDRESS_FILE"] = "$(LOG)/.master_address"
os.environ["_condor_COLLECTOR_ADDRESS_FILE"] = "$(LOG)/.collector_address"
os.environ["_condor_SCHEDD_ADDRESS_FILE"] = "$(LOG)/.schedd_address"
os.environ["_condor_STARTD_ADDRESS_FILE"] = "$(LOG)/.startd_address"
os.environ["_condor_NEGOTIATOR_ADDRESS_FILE"] = "$(LOG)/.negotiator_address"
# Various required attributes for the startd
os.environ["_condor_START"] = "TRUE"
os.environ["_condor_SUSPEND"] = "FALSE"
os.environ["_condor_CONTINUE"] = "TRUE"
os.environ["_condor_PREEMPT"] = "FALSE"
os.environ["_condor_KILL"] = "FALSE"
os.environ["_condor_WANT_SUSPEND"] = "FALSE"
os.environ["_condor_WANT_VACATE"] = "FALSE"
os.environ["_condor_MachineMaxVacateTime"] = "5"
htcondor.reload_config()
htcondor.SecMan().invalidateAllSessions()
def __exit__(self, exc_type, exc_val, exc_tb):
if self.previous_value is not None:
set_env_var("CONDOR_CONFIG", self.previous_value)
else:
unset_env_var("CONDOR_CONFIG")
htcondor.reload_config()
def fetch(self, constraint=None, format_list=None, condor_config=None):
"""
Fetch resource classads and return a list of evaluated classads
"""
results = []
constraint = bindings_friendly_constraint(constraint)
attrs = bindings_friendly_attrs(format_list)
adtype = resource_str_to_py_adtype(self.resource_str)
try:
old_condor_config_env = os.environ.get('CONDOR_CONFIG')
if condor_config and os.path.exists(condor_config):
os.environ['CONDOR_CONFIG'] = condor_config
htcondor.reload_config()
if self.pool_name:
collector = htcondor.Collector(str(self.pool_name))
else:
collector = htcondor.Collector()
classads = collector.query(adtype, constraint, attrs)
results = eval_classad_expr(classads, format_list=format_list)
except Exception:
p = 'default'
if self.pool_name is not None:
p = self.pool_name
logger.exception(f"Error querying pool {p} using python bindings")
raise QueryError()
finally:
if old_condor_config_env:
os.environ['CONDOR_CONFIG'] = old_condor_config_env
return results
def fetch_using_bindings(self, constraint=None, format_list=None):
"""
Fetch the results using htcondor-python bindings
"""
results_dict = {}
constraint = bindings_friendly_constraint(constraint)
attrs = bindings_friendly_attrs(format_list)
adtype = resource_str_to_py_adtype(self.resource_str)
self.security_obj.save_state()
try:
self.security_obj.enforce_requests()
htcondor.reload_config()
if self.pool_name:
collector = htcondor.Collector(str(self.pool_name))
else:
collector = htcondor.Collector()
results = collector.query(adtype, constraint, attrs)
results_dict = list2dict(results, self.group_attribute)
except Exception as ex:
p = 'default'
if self.pool_name is not None:
p = self.pool_name
err_str = 'Error querying pool %s using python bindings: %s' % (p, ex)
raise PBError(err_str), None, sys.exc_info()[2]
finally:
self.security_obj.restore_state()
return results_dict
def launch_daemons(self, daemons=["MASTER", "COLLECTOR"], config={}):
makedirs_ignore_exist(htcondor.param["LOG"])
makedirs_ignore_exist(htcondor.param["LOCK"])
makedirs_ignore_exist(htcondor.param["EXECUTE"])
makedirs_ignore_exist(htcondor.param["SPOOL"])
makedirs_ignore_exist(htcondor.param["RUN"])
remove_ignore_missing(htcondor.param["MASTER_ADDRESS_FILE"])
remove_ignore_missing(htcondor.param["COLLECTOR_ADDRESS_FILE"])
remove_ignore_missing(htcondor.param["SCHEDD_ADDRESS_FILE"])
for key, val in config.items():
os.environ["_condor_%s" % key] = val
if "COLLECTOR" in daemons:
os.environ["_condor_PORT"] = "9622"
os.environ["_condor_COLLECTOR_ARGS"] = "-port $(PORT)"
os.environ["_condor_COLLECTOR_HOST"] = "$(CONDOR_HOST):$(PORT)"
if 'MASTER' not in daemons:
daemons.append('MASTER')
os.environ["_condor_DAEMON_LIST"] = ", ".join(daemons)
htcondor.reload_config()
self.pid = os.fork()
if not self.pid:
try:
try:
os.execvp("condor_master", ["condor_master", "-f"])
except:
e = sys.exc_info()[1]
print(str(e))
finally:
os._exit(1)
global master_pid
master_pid = self.pid
for daemon in daemons:
self.waitLocalDaemon(daemon)
def get_htcondor_config(name="ADSTASH"):
htcondor.reload_config()
p = htcondor.param
conf = {
"sample_interval": p.get(f"{name}_SAMPLE_INTERVAL"),
"checkpoint_file": p.get(f"{name}_CHECKPOINT_FILE"),
"log_file": p.get(f"{name}_LOG"),
"debug_levels": p.get(f"{name}_DEBUG"),
"threads": p.get(f"{name}_NUM_THREADS"),
"collectors": p.get(f"{name}_READ_POOLS"),
"schedds": p.get(f"{name}_READ_SCHEDDS"),
"startds": p.get(f"{name}_READ_STARTDS"),
"schedd_history": p.get(f"{name}_SCHEDD_HISTORY"),
"startd_history": p.get(f"{name}_STARTD_HISTORY"),
"schedd_history_max_ads": p.get(f"{name}_SCHEDD_HISTORY_MAX_ADS"),
"startd_history_max_ads": p.get(f"{name}_STARTD_HISTORY_MAX_ADS"),
"schedd_history_timeout": p.get(f"{name}_SCHEDD_HISTORY_TIMEOUT"),
"startd_history_timeout": p.get(f"{name}_STARTD_HISTORY_TIMEOUT"),
"es_host": p.get(f"{name}_ES_HOST"),
"es_username": p.get(f"{name}_ES_USERNAME"),
"es_password_file": p.get(f"{name}_ES_PASSWORD_FILE"),
"es_use_https": p.get(f"{name}_ES_USE_HTTPS"),
"es_timeout": p.get(f"{name}_ES_TIMEOUT"),
"es_bunch_size": p.get(f"{name}_ES_BUNCH_SIZE"),
"es_index_name": p.get(f"{name}_ES_INDEX_NAME"),
"ca_certs": p.get(f"{name}_CA_CERTS"),
}
# Convert debug level
if conf.get("debug_levels") is not None and conf.get("debug_levels") != "":
conf["log_level"] = debug2level(conf["debug_levels"])
# Grab password from password file
if conf.get("es_password_file"
) is not None and conf.get("es_password_file") != "":
passwd = Path(conf["es_password_file"])
try:
with passwd.open() as f:
conf["es_password"] = str(f.read(4096)).split("\n")[0]
if conf["es_password"] == "":
logging.error(f"Got empty string from password file {passwd}")
except Exception:
logging.exception(
f"Fatal error while trying to read password file {passwd}")
# For schedds and startds, "*" is shorthand for all (which is the default)
for conf_key in ["schedds", "startds"]:
if conf.get(conf_key) == "*":
del conf[conf_key]
# remove None values
conf = {k: v for k, v in conf.items() if v is not None}
# normalize values
conf = normalize_config_types(conf)
return conf
def setUp(self):
self.pid = -1
to_delete = [i for i in os.environ if i.lower().startswith("_condor_")]
for key in to_delete:
del os.environ[key]
os.environ["_condor_MASTER"] = os.path.join(
os.getcwd(), "../condor_master.V6/condor_master")
os.environ["_condor_COLLECTOR"] = os.path.join(
os.getcwd(), "../condor_collector.V6/condor_collector")
os.environ["_condor_SCHEDD"] = os.path.join(
os.getcwd(), "../condor_schedd.V6/condor_schedd")
os.environ["_condor_PROCD"] = os.path.join(
os.getcwd(), "../condor_procd/condor_procd")
os.environ["_condor_STARTD"] = os.path.join(
os.getcwd(), "../condor_startd.V6/condor_startd")
os.environ["_condor_STARTER"] = os.path.join(
os.getcwd(), "../condor_starter.V6.1/condor_starter")
os.environ["_condor_NEGOTIATOR"] = os.path.join(
os.getcwd(), "../condor_negotiator.V6/condor_negotiator")
os.environ["_condor_SHADOW"] = os.path.join(
os.getcwd(), "../condor_shadow.V6.1/condor_shadow")
os.environ["_condor_SHARED_PORT"] = os.path.join(
os.getcwd(), "../condor_shared_port/condor_shared_port")
os.environ["_condor_CONDOR_HOST"] = socket.getfqdn()
os.environ["_condor_LOCAL_DIR"] = testdir
os.environ["_condor_LOG"] = '$(LOCAL_DIR)/log'
os.environ["_condor_LOCK"] = '$(LOCAL_DIR)/lock'
os.environ["_condor_RUN"] = '$(LOCAL_DIR)/run'
os.environ["_condor_COLLECTOR_NAME"] = "python_classad_tests"
os.environ["_condor_SCHEDD_NAME"] = "python_classad_tests"
os.environ["_condor_MASTER_ADDRESS_FILE"] = "$(LOG)/.master_address"
os.environ[
"_condor_COLLECTOR_ADDRESS_FILE"] = "$(LOG)/.collector_address"
os.environ["_condor_SCHEDD_ADDRESS_FILE"] = "$(LOG)/.schedd_address"
os.environ["_condor_STARTD_ADDRESS_FILE"] = "$(LOG)/.startd_address"
os.environ["_condor_STARTD_DEBUG"] = "D_FULLDEBUG"
os.environ["_condor_STARTER_DEBUG"] = "D_FULLDEBUG"
os.environ["_condor_SHADOW_DEBUG"] = "D_FULLDEBUG|D_MACHINE"
os.environ[
"_condor_NEGOTIATOR_ADDRESS_FILE"] = "$(LOG)/.negotiator_address"
os.environ["_condor_NEGOTIATOR_CYCLE_DELAY"] = "1"
os.environ["_condor_NEGOTIATOR_INTERVAL"] = "1"
os.environ["_condor_SCHEDD_INTERVAL"] = "1"
os.environ["_condor_SCHEDD_MIN_INTERVAL"] = "1"
os.environ["_condor_CONDOR_FSYNC"] = "FALSE"
# Various required attributes for the startd
os.environ["_condor_START"] = "TRUE"
os.environ["_condor_SUSPEND"] = "FALSE"
os.environ["_condor_CONTINUE"] = "TRUE"
os.environ["_condor_PREEMPT"] = "FALSE"
os.environ["_condor_KILL"] = "FALSE"
os.environ["_condor_WANT_SUSPEND"] = "FALSE"
os.environ["_condor_WANT_VACATE"] = "FALSE"
os.environ["_condor_MachineMaxVacateTime"] = "5"
os.environ["_condor_JOB_INHERITS_STARTER_ENVIRONMENT"] = "TRUE"
htcondor.reload_config()
htcondor.SecMan().invalidateAllSessions()
def htcondor_full_reload():
HTCONDOR_ENV_PREFIX = "_CONDOR_"
HTCONDOR_ENV_PREFIX_LEN = len(HTCONDOR_ENV_PREFIX) # len of _CONDOR_ = 8
if not USE_HTCONDOR_PYTHON_BINDINGS:
return
# Reload configuration reading CONDOR_CONFIG from the environment
htcondor.reload_config()
# _CONDOR_ variables need to be added manually to _Params
for i in os.environ:
if i.startswith(HTCONDOR_ENV_PREFIX):
htcondor.param[i[HTCONDOR_ENV_PREFIX_LEN:]] = os.environ[i]
def SetupLocalEnvironment(self, params=None):
Utils.MakedirsIgnoreExist(self._local_dir)
Utils.MakedirsIgnoreExist(self._execute_path)
Utils.MakedirsIgnoreExist(self._log_path)
Utils.MakedirsIgnoreExist(self._lock_path)
Utils.MakedirsIgnoreExist(self._run_path)
Utils.MakedirsIgnoreExist(self._spool_path)
# Create a config file in this test's local directory based on existing
# config settings
os.system("condor_config_val -write:up " + self._local_config)
# Add whatever internal config values we need
config = "LOCAL_DIR = " + self._local_path + "\n"
config += "EXECUTE = " + self._execute_path + "\n"
config += "LOCK = " + self._lock_path + "\n"
config += "RUN = " + self._run_path + "\n"
config += "SPOOL = " + self._spool_path + "\n"
config += "COLLECTOR_HOST = $(CONDOR_HOST):0\n"
config += "MASTER_ADDRESS_FILE = $(LOG)/.master_address\n"
config += "COLLECTOR_ADDRESS_FILE = $(LOG)/.collector_address\n"
config += "SCHEDD_ADDRESS_FILE = $(SPOOL)/.schedd_address\n"
if Utils.IsWindows() is True:
config += "PROCD_ADDRESS = " + str(
htcondor.param["PROCD_ADDRESS"]) + str(os.getpid()) + "\n"
# Add any custom params
if params is not None:
for key in params:
config += key + " = " + params[key] + "\n"
config_file = open(self._local_config, "a")
config_file.write(config)
config_file.close()
# Set CONDOR_CONFIG to apply the changes we just wrote to file
self.SetCondorConfig()
# MRC: What does this function actually do?
htcondor.reload_config()
# Now that we have our config setup, delete any old files potentially left over
Utils.RemoveIgnoreMissing(htcondor.param["MASTER_ADDRESS_FILE"])
Utils.RemoveIgnoreMissing(htcondor.param["COLLECTOR_ADDRESS_FILE"])
Utils.RemoveIgnoreMissing(htcondor.param["SCHEDD_ADDRESS_FILE"])
def _condor_advertise(self,
classads,
collector_host=None,
update_ad_command=DEFAULT_UPDATE_AD_COMMAND):
"""
Advertise list of classads to the HTCondor Collector
:type ads: :obj:`list`
:type collector_host: :obj:`string`
"""
ads = classads
old_condor_config_env = os.environ.get("CONDOR_CONFIG")
try:
if self.condor_config and os.path.exists(self.condor_config):
os.environ["CONDOR_CONFIG"] = self.condor_config
htcondor.reload_config()
if self.x509_user_proxy and os.path.exists(self.x509_user_proxy):
os.environ["X509_USER_PROXY"] = self.x509_user_proxy
collector = None
if collector_host:
collector = htcondor.Collector(collector_host)
else:
collector_host = "default"
collector = htcondor.Collector()
self.logger.info(
f"Advertising {self.classad_type} classads to collector_host {collector_host}"
)
collector.advertise(ads, update_ad_command, True)
except Exception:
# TODO: We need to be more specific about the errors/exception
# For now just raise to get more info logged
col = "default"
if collector_host:
col = collector_host
self.logger.error(
f"Error running {update_ad_command} for {self.classad_type} classads to collector_host {col}"
)
# err_str = 'Error advertising with command %s to pool %s: %s' % (self.update_ad_command, col, ex)
# raise QueryError(err_str), None, sys.exc_info()[2]
raise
finally:
if old_condor_config_env:
os.environ["CONDOR_CONFIG"] = old_condor_config_env
def setUp(self):
self.pid = -1
to_delete = [i for i in os.environ if i.lower().startswith("_condor_")]
for key in to_delete: del os.environ[key]
os.environ["_condor_MASTER"] = os.path.join(os.getcwd(), "../condor_master.V6/condor_master")
os.environ["_condor_COLLECTOR"] = os.path.join(os.getcwd(), "../condor_collector.V6/condor_collector")
os.environ["_condor_SCHEDD"] = os.path.join(os.getcwd(), "../condor_schedd.V6/condor_schedd")
os.environ["_condor_PROCD"] = os.path.join(os.getcwd(), "../condor_procd/condor_procd")
os.environ["_condor_STARTD"] = os.path.join(os.getcwd(), "../condor_startd.V6/condor_startd")
os.environ["_condor_STARTER"] = os.path.join(os.getcwd(), "../condor_starter.V6.1/condor_starter")
os.environ["_condor_NEGOTIATOR"] = os.path.join(os.getcwd(), "../condor_negotiator.V6/condor_negotiator")
os.environ["_condor_SHADOW"] = os.path.join(os.getcwd(), "../condor_shadow.V6.1/condor_shadow")
os.environ["_condor_SHARED_PORT"] = os.path.join(os.getcwd(), "../condor_shared_port/condor_shared_port")
os.environ["_condor_CONDOR_HOST"] = socket.getfqdn()
os.environ["_condor_LOCAL_DIR"] = testdir
os.environ["_condor_LOG"] = '$(LOCAL_DIR)/log'
os.environ["_condor_LOCK"] = '$(LOCAL_DIR)/lock'
os.environ["_condor_RUN"] = '$(LOCAL_DIR)/run'
os.environ["_condor_COLLECTOR_NAME"] = "python_classad_tests"
os.environ["_condor_SCHEDD_NAME"] = "python_classad_tests"
os.environ["_condor_MASTER_ADDRESS_FILE"] = "$(LOG)/.master_address"
os.environ["_condor_COLLECTOR_ADDRESS_FILE"] = "$(LOG)/.collector_address"
os.environ["_condor_SCHEDD_ADDRESS_FILE"] = "$(LOG)/.schedd_address"
os.environ["_condor_STARTD_ADDRESS_FILE"] = "$(LOG)/.startd_address"
os.environ["_condor_STARTD_DEBUG"] = "D_FULLDEBUG"
os.environ["_condor_STARTER_DEBUG"] = "D_FULLDEBUG"
os.environ["_condor_SHADOW_DEBUG"] = "D_FULLDEBUG|D_MACHINE"
os.environ["_condor_NEGOTIATOR_ADDRESS_FILE"] = "$(LOG)/.negotiator_address"
os.environ["_condor_NEGOTIATOR_CYCLE_DELAY"] = "1"
os.environ["_condor_NEGOTIATOR_INTERVAL"] = "1"
os.environ["_condor_SCHEDD_INTERVAL"] = "1"
os.environ["_condor_SCHEDD_MIN_INTERVAL"] = "1"
os.environ["_condor_CONDOR_FSYNC"] = "FALSE"
# Various required attributes for the startd
os.environ["_condor_START"] = "TRUE"
os.environ["_condor_SUSPEND"] = "FALSE"
os.environ["_condor_CONTINUE"] = "TRUE"
os.environ["_condor_PREEMPT"] = "FALSE"
os.environ["_condor_KILL"] = "FALSE"
os.environ["_condor_WANT_SUSPEND"] = "FALSE"
os.environ["_condor_WANT_VACATE"] = "FALSE"
os.environ["_condor_MachineMaxVacateTime"] = "5"
os.environ["_condor_JOB_INHERITS_STARTER_ENVIRONMENT"] = "TRUE"
htcondor.reload_config()
htcondor.SecMan().invalidateAllSessions()
def config_val(attr):
"""Query HTCondor for the value of a configuration variable using the python
bindings if available, condor_config_val otherwise
"""
try:
import htcondor
# Necessary for checking config between different flavors of HTCondor
htcondor.reload_config()
try:
val = htcondor.param[attr]
except KeyError: # attr is undefined
val = None
except:
out, _, _ = core.check_system(('condor_config_val', attr),
'Failed to query for config variable: %s' % attr)
val = out.strip()
return val
def get(self, attribute=None):
"""GET handler"""
parser = reqparse.RequestParser(trim=True)
parser.add_argument("daemon",
choices=list(self.DAEMON_TYPES_MAP.keys()))
args = parser.parse_args()
param = None
if args.daemon:
daemon_ad = None
try:
daemon_ad = Collector().locate(
self.DAEMON_TYPES_MAP[args.daemon])
except (IOError, RuntimeError) as err:
abort(503,
message=FAIL_QUERY % {
"service": "collector",
"err": err
})
try:
param = RemoteParam(daemon_ad)
except (IOError, RuntimeError) as err:
abort(503,
message=FAIL_QUERY % {
"service": args.daemon,
"err": err
})
else:
htcondor.reload_config()
param = htcondor.param
param_lower = utils.deep_lcasekeys(param)
if attribute:
if not utils.validate_attribute(attribute):
abort(400, message=BAD_ATTRIBUTE_OR_PROJECTION)
if attribute:
try:
return param_lower[attribute.lower()]
except KeyError:
abort(404, message=NO_ATTRIBUTE)
return param_lower
def setUp(self):
self.pid = -1
os.environ["_condor_MASTER"] = os.path.join(os.getcwd(), "../condor_master.V6/condor_master")
os.environ["_condor_COLLECTOR"] = os.path.join(os.getcwd(), "../condor_collector.V6/condor_collector")
os.environ["_condor_SCHEDD"] = os.path.join(os.getcwd(), "../condor_schedd.V6/condor_schedd")
os.environ["_condor_PROCD"] = os.path.join(os.getcwd(), "../condor_procd/condor_procd")
os.environ["_condor_STARTD"] = os.path.join(os.getcwd(), "../condor_startd.V6/condor_startd")
os.environ["_condor_STARTER"] = os.path.join(os.getcwd(), "../condor_starter.V6.1/condor_starter")
os.environ["_condor_NEGOTIATOR"] = os.path.join(os.getcwd(), "../condor_negotiator.V6/condor_negotiator")
os.environ["_condor_SHADOW"] = os.path.join(os.getcwd(), "../condor_shadow.V6.1/condor_shadow")
os.environ["_condor_STARTER.PLUGINS"] = os.path.join(os.getcwd(), "../condor_contrib/lark/lark-plugin.so")
os.environ["_condor_USE_NETWORK_NAMESPACES"] = "TRUE"
#os.environ["_condor_LARK_NETWORK_ACCOUNTING"] = "TRUE"
#now make the default configuration to be "bridge"
#os.environ["_condor_STARTD_ATTRS"] = "LarkNetworkType, LarkAddressType, LarkBridgeDevice"
#os.environ["_condor_LarkNetworkType"] = "bridge"
#os.environ["_condor_LarkNetBridgeDevice"] = "eth0"
#os.environ["_condor_LarkAddressType"] = "dhcp"
os.environ["_condor_CONDOR_HOST"] = socket.getfqdn()
os.environ["_condor_LOCAL_DIR"] = testdir
os.environ["_condor_LOG"] = '$(LOCAL_DIR)/log'
os.environ["_condor_LOCK"] = '$(LOCAL_DIR)/lock'
os.environ["_condor_RUN"] = '$(LOCAL_DIR)/run'
os.environ["_condor_COLLECTOR_NAME"] = "python_classad_tests"
os.environ["_condor_SCHEDD_NAME"] = "python_classad_tests"
os.environ["_condor_MASTER_ADDRESS_FILE"] = "$(LOG)/.master_address"
os.environ["_condor_COLLECTOR_ADDRESS_FILE"] = "$(LOG)/.collector_address"
os.environ["_condor_SCHEDD_ADDRESS_FILE"] = "$(LOG)/.schedd_address"
os.environ["_condor_STARTD_ADDRESS_FILE"] = "$(LOG)/.startd_address"
os.environ["_condor_NEGOTIATOR_ADDRESS_FILE"] = "$(LOG)/.negotiator_address"
# Various required attributes for the startd
os.environ["_condor_START"] = "TRUE"
os.environ["_condor_SUSPEND"] = "FALSE"
os.environ["_condor_CONTINUE"] = "TRUE"
os.environ["_condor_PREEMPT"] = "FALSE"
os.environ["_condor_KILL"] = "FALSE"
os.environ["_condor_WANT_SUSPEND"] = "FALSE"
os.environ["_condor_WANT_VACATE"] = "FALSE"
# Remember to check the correctness of network policy script path defined
os.environ["_condor_STARTER_NETWORK_POLICY_SCRIPT_PATH"] = os.path.join(os.getcwd(), "../condor_contrib/lark/LarkNetworkPolicy/lark_network_policy.py")
os.environ["_condor_STARTER_DEBUG"] = "D_FULLDEBUG"
htcondor.reload_config()
htcondor.SecMan().invalidateAllSessions()
def _condor_advertise(self,
classads,
collector_host=None,
update_ad_command=DEFAULT_UPDATE_AD_COMMAND):
"""
Advertise list of classads to the HTCondor Collector
:type ads: :obj:`list`
:type collector_host: :obj:`string`
"""
ads = classads
old_condor_config_env = os.environ.get('CONDOR_CONFIG')
try:
if self.condor_config and os.path.exists(self.condor_config):
os.environ['CONDOR_CONFIG'] = self.condor_config
htcondor.reload_config()
if self.x509_user_proxy and os.path.exists(self.x509_user_proxy):
os.environ['X509_USER_PROXY'] = self.x509_user_proxy
collector = None
if collector_host:
collector = htcondor.Collector(collector_host)
else:
collector_host = 'default'
collector = htcondor.Collector()
self.logger.info(
f"Advertising {self.classad_type} classads to collector_host {collector_host}"
)
collector.advertise(ads, update_ad_command, True)
except Exception:
col = 'default'
if collector_host:
col = collector_host
self.logger.exception(
f"Error running {update_ad_command} for {self.classad_type} classads to collector_host {col}"
)
raise
finally:
if old_condor_config_env:
os.environ['CONDOR_CONFIG'] = old_condor_config_env
def fetch(self, constraint=None, format_list=None, condor_config=None):
"""
Fetch job classads
"""
results = []
constraint = bindings_friendly_constraint(constraint)
attrs = bindings_friendly_attrs(format_list)
try:
old_condor_config_env = os.environ.get('CONDOR_CONFIG')
if condor_config and os.path.exists(condor_config):
os.environ['CONDOR_CONFIG'] = condor_config
htcondor.reload_config()
if self.pool_name:
collector = htcondor.Collector(str(self.pool_name))
else:
collector = htcondor.Collector()
if self.schedd_name is None:
schedd = htcondor.Schedd()
else:
schedd = htcondor.Schedd(
collector.locate(htcondor.DaemonTypes.Schedd,
self.schedd_name))
classads = schedd.query(constraint, attrs)
#results_dict = list2dict(results, self.group_attr)
results = eval_classad_expr(classads, format_list=format_list)
except Exception as ex:
s = 'default'
if self.schedd_name is not None:
s = self.schedd_name
p = 'default'
if self.pool_name is not None:
p = self.pool_name
err_str = 'Error querying schedd %s in pool %s using python bindings: %s' % (
s, p, ex)
raise QueryError(err_str).with_traceback(sys.exc_info()[2])
finally:
if old_condor_config_env:
os.environ['CONDOR_CONFIG'] = old_condor_config_env
return results
def executeAll(self, joblist=None, attributes=None, values=None):
""" Given equal sized lists of job ids, attributes and values,
executes in one large transaction a single qedit for each job.
"""
joblist = joblist or []
attributes = attributes or []
values = values or []
if not (len(joblist) == len(attributes) == len(values)):
raise QueryError(
"Arguments to QEdit.executeAll should have the same length")
try:
htcondor.reload_config()
if self.pool_name:
collector = htcondor.Collector(str(self.pool_name))
else:
collector = htcondor.Collector()
if self.schedd_name:
schedd_ad = collector.locate(htcondor.DaemonTypes.Schedd,
self.schedd_name)
schedd = htcondor.Schedd(schedd_ad)
else:
schedd = htcondor.Schedd()
with schedd.transaction() as _:
for jobid, attr, val in zip(joblist, attributes, values):
schedd.edit([jobid], attr, classad.quote(val))
except Exception as ex:
s = 'default'
if self.schedd_name is not None:
s = self.schedd_name
p = 'default'
if self.pool_name is not None:
p = self.pool_name
try:
j1 = jobid
j2 = attr
j3 = val
except:
j1 = j2 = j3 = 'unknown'
err_str = 'Error querying schedd %s in pool %s using python bindings (qedit of job/attr/val %s/%s/%s): %s' % (
s, p, j1, j2, j3, ex)
raise QueryError(err_str)
def executeAll(self, joblist=None, attributes=None, values=None):
""" Given equal sized lists of job ids, attributes and values,
executes in one large transaction a single qedit for each job.
"""
joblist = joblist or []
attributes = attributes or []
values = values or []
if not (len(joblist) == len(attributes) == len(values)):
raise QueryError("Arguments to QEdit.executeAll should have the same length")
try:
htcondor.reload_config()
if self.pool_name:
collector = htcondor.Collector(str(self.pool_name))
else:
collector = htcondor.Collector()
if self.schedd_name:
schedd_ad = collector.locate(htcondor.DaemonTypes.Schedd,
self.schedd_name)
schedd = htcondor.Schedd(schedd_ad)
else:
schedd = htcondor.Schedd()
with schedd.transaction() as _:
for jobid, attr, val in zip(joblist, attributes, values):
schedd.edit([jobid], attr, classad.quote(val))
except Exception as ex:
s = 'default'
if self.schedd_name is not None:
s = self.schedd_name
p = 'default'
if self.pool_name is not None:
p = self.pool_name
try:
j1 = jobid
j2 = attr
j3 = val
except:
j1 = j2 = j3 = 'unknown'
err_str = 'Error querying schedd %s in pool %s using python bindings (qedit of job/attr/val %s/%s/%s): %s' % (s, p, j1, j2, j3, ex)
raise QueryError(err_str)
def fetch_using_bindings(self, constraint=None, format_list=None):
"""
Fetch the results using htcondor-python bindings
"""
results_dict = {}
constraint = bindings_friendly_constraint(constraint)
attrs = bindings_friendly_attrs(format_list)
self.security_obj.save_state()
try:
self.security_obj.enforce_requests()
htcondor.reload_config()
if self.pool_name:
collector = htcondor.Collector(str(self.pool_name))
else:
collector = htcondor.Collector()
if self.schedd_name is None:
schedd = htcondor.Schedd()
else:
schedd_ad = collector.locate(htcondor.DaemonTypes.Schedd,
self.schedd_name)
schedd = htcondor.Schedd(schedd_ad)
results = schedd.query(constraint, attrs)
results_dict = list2dict(results, self.group_attribute)
except Exception as ex:
s = 'default'
if self.schedd_name is not None:
s = self.schedd_name
p = 'default'
if self.pool_name is not None:
p = self.pool_name
err_str = 'Error querying schedd %s in pool %s using python bindings: %s' % (s, p, ex)
raise PBError(err_str), None, sys.exc_info()[2]
finally:
self.security_obj.restore_state()
return results_dict
set([x for x in entry.keys() if x.startswith('eval_set_')]))
if no_effect_attr:
warn(
"%s in JOB_ROUTER_ENTRIES may not have any effect. Use the 'set_ prefix instead."
% ', '.join(no_effect_attr))
# Warn users on OSG CEs if osg-configure has not been run
if is_osg:
try:
htcondor.param['OSG_CONFIGURED']
except KeyError:
warn("osg-configure has not been run, degrading the functionality "
"of the CE. Please run 'osg-configure -c' and restart condor-ce.")
# Ensure that HTCondor back-ends have QUEUE_SUPER_USER_MAY_IMPERSONATE set correctly
try:
htcondor.param['JOB_ROUTER_SCHEDD2_NAME']
except KeyError:
pass
else:
os.environ['CONDOR_CONFIG'] = '/etc/condor/condor_config'
htcondor.reload_config()
su_attr = 'QUEUE_SUPER_USER_MAY_IMPERSONATE'
if htcondor.param.get(su_attr, '') != '.*':
error(
"HTCondor batch system is improperly configured for use with HTCondor CE. "
"Please verify that '%s = .*' is set in your HTCondor configuration."
% su_attr)
finally:
os.environ['CONDOR_CONFIG'] = '/etc/condor-ce/condor_config'
def _SetupLocalEnvironment(self):
Utils.MakedirsIgnoreExist(self._local_dir)
Utils.MakedirsIgnoreExist(self._execute_path)
Utils.MakedirsIgnoreExist(self._log_path)
Utils.MakedirsIgnoreExist(self._lock_path)
Utils.MakedirsIgnoreExist(self._run_path)
Utils.MakedirsIgnoreExist(self._spool_path)
# Create a config file in this test's local directory based on existing
# config settings
os.system("condor_config_val -write:up " + self._local_config)
# Add whatever internal config values we need
config = """
#
# From PersonalCondor
#
"""
config += "LOCAL_DIR = " + self._local_path + "\n"
config += "EXECUTE = " + self._execute_path + "\n"
config += "LOCK = " + self._lock_path + "\n"
config += "LOG = " + self._log_path + "\n"
config += "RUN = " + self._run_path + "\n"
config += "SPOOL = " + self._spool_path + "\n"
config += "COLLECTOR_HOST = $(CONDOR_HOST):0\n"
config += "MASTER_ADDRESS_FILE = $(LOG)/.master_address\n"
config += "COLLECTOR_ADDRESS_FILE = $(LOG)/.collector_address\n"
config += "SCHEDD_ADDRESS_FILE = $(SPOOL)/.schedd_address\n"
if Utils.IsWindows() is True:
# This call to htcondor.param() will return the correct value iff
# nobody set CONDOR_CONFIG without calling htcondor.reload_config();
# it's not clear if it's better for us to call that before calling
# condor_config_val above, or if to avoid perturbing the system
# any more than necessary.
config += "PROCD_ADDRESS = " + str(
htcondor.param["PROCD_ADDRESS"]) + str(os.getpid()) + "\n"
config += """
#
# Default params
#
"""
for key in PersonalCondor.default_params:
config += key + " = " + str(
PersonalCondor.default_params[key]) + "\n"
# Add any custom params
if self._params is not None:
config += """
#
# Custom params
#
"""
for key in self._params:
if self._params[key] is None:
config += key + "\n"
else:
config += key + " = " + str(self._params[key]) + "\n"
if self._ordered_params is not None:
config += """
#
# Ordered params
#
"""
config += self._ordered_params
config_file = open(self._local_config, "a")
config_file.write(config)
config_file.close()
# Set CONDOR_CONFIG to apply the changes we just wrote to file
self.SetCondorConfig()
# If we didn't do this, htcondor.param[] would return results from the
# old CONDOR_CONFIG, which most would find astonishing (since most of
# the time, there will only be a single relevant instance).
htcondor.reload_config()
# Now that we have our config setup, delete any old files potentially left over
Utils.RemoveIgnoreMissing(htcondor.param["MASTER_ADDRESS_FILE"])
Utils.RemoveIgnoreMissing(htcondor.param["COLLECTOR_ADDRESS_FILE"])
Utils.RemoveIgnoreMissing(htcondor.param["SCHEDD_ADDRESS_FILE"])
def main():
"""Main function
"""
is_osg = htcondor.param.get('OSG_CONFIGURE_PRESENT',
'').lower() in ('true', 'yes', '1')
# Create dict whose values are lists of ads specified in the relevant JOB_ROUTER_* variables
parsed_jr_ads = {}
for attr in ['JOB_ROUTER_DEFAULTS', 'JOB_ROUTER_ENTRIES']:
try:
config_val = htcondor.param[attr]
except KeyError:
error("Missing required %s configuration value" % attr)
# store the ads (iterating through ClassAdStringIterator consumes them)
try:
parsed_jr_ads[attr] = list(classad.parseAds(config_val))
except ValueError:
# We shouldn't ever get here since classad.parseAds() only raises ValueError when it's given
# non-string/non-file output and htcondor.param shouldn't contain such values
error("Failed to parse %s configuration value" % attr)
# If JRD or JRE can't be parsed, the job router can't function
if not parsed_jr_ads[attr]:
error("Could not read %s in the HTCondor-CE configuration." % attr)
if attr == "JOB_ROUTER_ENTRIES":
# Warn about routes we can find in the config that don't result in valid ads
malformed_entry_names = find_malformed_entries(config_val)
if malformed_entry_names:
warn(
"Could not read JOB_ROUTER_ENTRIES in the HTCondor-CE configuration. "
+ "Failed to parse the following routes: %s" %
', '.join(malformed_entry_names))
# Warn about routes specified by JOB_ROUTER_ROUTE_NAMES that don't appear in the parsed JRE.
# The job router can function this way but it's likely a config error
route_order = htcondor.param.get('JOB_ROUTER_ROUTE_NAMES', '')
if route_order:
missing_route_def = set(route_order).difference(
set(parse_route_names(config_val)))
if missing_route_def:
warn(
"The following are specified in JOB_ROUTER_ROUTE_NAMES "
"but cannot be found in JOB_ROUTER_ENTRIES: %s" %
', '.join(missing_route_def))
# Find all eval_set_ attributes in the JOB_ROUTER_DEFAULTS
eval_set_defaults = set([
x.lstrip('eval_')
for x in parsed_jr_ads['JOB_ROUTER_DEFAULTS'][0].keys()
if x.startswith('eval_set_')
])
# Find all default_ attributes used in expressions in the JOB_ROUTER_DEFAULTS
default_attr = set([
re.sub(r'.*(default_\w*).*', 'eval_set_\\1', str(x))
for x in parsed_jr_ads['JOB_ROUTER_DEFAULTS'][0].values()
if isinstance(x, classad.ExprTree) and "default_" in str(x)
])
for entry in parsed_jr_ads['JOB_ROUTER_ENTRIES']:
# Warn users if they've set_ attributes that would be overriden by eval_set in the JOB_ROUTER_DEFAULTS
overriden_attr = eval_set_defaults.intersection(set(entry.keys()))
if overriden_attr:
warn(
"%s in JOB_ROUTER_ENTRIES will be overriden by the JOB_ROUTER_DEFAULTS."
% ', '.join(overriden_attr) +
" Use the 'eval_set_' prefix instead.")
# Ensure that users don't set the job environment in the Job Router
if is_osg and any(x.endswith('environment') for x in entry.keys()):
error(
"Do not use the Job Router to set the environment. Place variables under "
+ "[Local Settings] in /etc/osg/config.d/40-localsettings.ini")
# Warn users about eval_set_ default attributes in the ENTRIES since their
# evaluation may occur after the eval_set_ expressions containg them in the
# JOB_ROUTER_DEFAULTS
no_effect_attr = default_attr.intersection(
set([x for x in entry.keys() if x.startswith('eval_set_')]))
if no_effect_attr:
warn("%s in JOB_ROUTER_ENTRIES " % ', '.join(no_effect_attr) +
"may not have any effect. Use the 'set_' prefix instead.")
# Warn users on OSG CEs if osg-configure has not been run
if is_osg:
try:
htcondor.param['OSG_CONFIGURED']
except KeyError:
warn(
"osg-configure has not been run, degrading the functionality "
+
"of the CE. Please run 'osg-configure -c' and restart condor-ce."
)
# Ensure that HTCondor back-ends have QUEUE_SUPER_USER_MAY_IMPERSONATE set correctly
try:
htcondor.param['JOB_ROUTER_SCHEDD2_NAME']
except KeyError:
pass
else:
os.environ['CONDOR_CONFIG'] = '/etc/condor/condor_config'
htcondor.reload_config()
su_attr = 'QUEUE_SUPER_USER_MAY_IMPERSONATE'
if htcondor.param.get(su_attr, '') != '.*':
error(
"HTCondor batch system is improperly configured for use with HTCondor CE. "
+
"Please verify that '%s = .*' is set in your HTCondor configuration."
% su_attr)
finally:
os.environ['CONDOR_CONFIG'] = '/etc/condor-ce/condor_config'
def set(self):
"""Set ``CONDOR_CONFIG`` and tell HTCondor to reconfigure."""
self.previous_value = os.environ.get("CONDOR_CONFIG", None)
_set_env_var("CONDOR_CONFIG", str(self.config_file))
htcondor.reload_config()
