def __init__(self, config, logger=None):
if not logger:
self.logger = logging.getLogger(__name__)
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
"%(asctime)s:%(levelname)s:%(module)s %(message)s")
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.logger.setLevel(logging.DEBUG)
else:
# do not use BaseRecurringAction logger but create a new logger
# which writes to config.TaskWorker.logsDir/taks/recurring/RenewRemoveProxies_YYMMDD-HHMM.log
self.logger = logging.getLogger('RenewRemoteProxies')
logDir = config.TaskWorker.logsDir + '/tasks/recurring/'
if not os.path.exists(logDir):
os.makedirs(logDir)
timeStamp = time.strftime('%y%m%d-%H%M', time.localtime())
logFile = 'RenewRemoveProxies_' + timeStamp + '.log'
handler = logging.FileHandler(logDir + logFile)
formatter = logging.Formatter(
'%(asctime)s:%(levelname)s:%(module)s:%(message)s')
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.config = config
self.pool = ''
self.schedds = []
self.restHost = None
self.dbInstance = None
htcondor.param['TOOL_DEBUG'] = 'D_FULLDEBUG D_SECURITY'
if 'CRAB3_DEBUG' in os.environ and hasattr(htcondor, 'enable_debug'):
htcondor.enable_debug()
def cli():
args = parse_args()
if args.debug:
warning("Debug mode enabled...")
htcondor.enable_debug()
try:
return watch_q(
users=args.users,
cluster_ids=args.clusters,
event_logs=args.files,
batches=args.batches,
collector=args.collector,
schedd=args.schedd,
exit_conditions=args.exit,
group_by=args.groupby,
table=args.table,
progress_bar=args.progress,
row_progress_bar=args.row_progress,
summary=args.summary,
summary_type=args.summary_type,
updated_at=args.updated_at,
color=args.color,
refresh=args.refresh,
abbreviate_path_components=args.abbreviate,
)
except Exception as e:
if args.debug:
raise
error(
"Unhandled error: {}. Re-run with -debug for a full stack trace.".
format(e))
sys.exit(1)
def cli():
args = parse_args()
if args.debug:
print("Enabling HTCondor debug output...", file=sys.stderr)
htcondor.enable_debug()
return watch_q(
users=args.users,
cluster_ids=args.clusters,
event_logs=args.files,
batches=args.batches,
collector=args.collector,
schedd=args.schedd,
exit_conditions=args.exit,
group_by=args.groupby,
table=args.table,
progress_bar=args.progress,
summary=args.summary,
summary_type=args.summary_type,
updated_at=args.updated_at,
color=args.color,
refresh=args.refresh,
abbreviate_path_components=args.abbreviate,
)
def main():
args = parse_args()
if args.verbose:
# HTCondor library logging setup
htcondor.param["TOOL_DEBUG"] = "D_FULLDEBUG D_SECURITY"
htcondor.enable_debug()
# Python logging setup
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(stream=sys.stderr)
handler.setLevel(logging.DEBUG)
handler.setFormatter(
logging.Formatter(
"%(asctime)s [%(levelname)s] %(filename)s:%(lineno)s ~ %(message)s"
))
logger.addHandler(handler)
if not SOURCE_CHECK.match(args.host):
error(
"The requested hostname must be composed of only alphabetical characters (A-Z, a-z), digits (0-9), periods (.), and dashes (-). It may not begin with a digit."
)
# TODO: Not clear this is necessary for Docker.
#if not is_admin():
# error(
# "This command must be run as root (on Linux/Mac) or as an administrator (on Windows)"
# )
logger.debug('Setting SEC_CLIENT_AUTHENTICATION_METHODS to "SSL"')
htcondor.param["SEC_CLIENT_AUTHENTICATION_METHODS"] = "SSL"
htcondor.param["SEC_CLIENT_ENCRYPTION"] = "REQUIRED"
# TODO: temporary fix for https://github.com/HTPhenotyping/registration/issues/17
if htcondor.param[
"AUTH_SSL_CLIENT_CAFILE"] == "/etc/ssl/certs/ca-bundle.crt":
htcondor.param[
"AUTH_SSL_CLIENT_CAFILE"] = "/etc/ssl/certs/ca-certificates.crt"
success = request_token(pool=args.pool,
resource=args.host,
scopes=args.scope,
local_dir=args.local_dir)
if not success:
error("Failed to complete the token request workflow.")
reconfig()
print("Registration of resource {} is complete!".format(args.host))
def cli():
args = parse_args()
if args.debug:
print("Enabling HTCondor debug output...", file=sys.stderr)
htcondor.enable_debug()
return watch_q(
users=args.users,
cluster_ids=args.clusters,
event_logs=args.files,
batches=args.batches,
exit_conditions=args.exit,
abbreviate_path_components=args.abbreviate,
groupby=args.groupby,
)
def __init__(self, config, resthost, resturi, logger=None):
if not logger:
self.logger = logging.getLogger(__name__)
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s %(message)s")
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.logger.setLevel(logging.DEBUG)
else:
self.logger = logger
self.resturi = resturi
self.resthost = resthost
self.config = config
self.pool = ''
self.schedds = []
htcondor.param['TOOL_DEBUG'] = 'D_FULLDEBUG D_SECURITY'
if 'CRAB3_DEBUG' in os.environ and hasattr(htcondor, 'enable_debug'):
htcondor.enable_debug()
def __init__(self, config, resthost, resturi, logger=None):
if not logger:
self.logger = logging.getLogger(__name__)
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s %(message)s")
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.logger.setLevel(logging.DEBUG)
else:
self.logger = logger
self.resturi = resturi
self.resthost = resthost
self.config = config
self.pool = ''
self.schedds = []
htcondor.param['TOOL_DEBUG'] = 'D_FULLDEBUG D_SECURITY'
if 'CRAB3_DEBUG' in os.environ and hasattr(htcondor, 'enable_debug'):
htcondor.enable_debug()
def testRemoteSetParam(self):
os.environ["_condor_SETTABLE_ATTRS_READ"] = "FOO"
os.environ["_condor_ENABLE_RUNTIME_CONFIG"] = "TRUE"
self.launch_daemons(["COLLECTOR"])
del os.environ["_condor_SETTABLE_ATTRS_READ"]
#htcondor.param["TOOL_DEBUG"] = "D_NETWORK|D_SECURITY"
htcondor.enable_debug()
coll = htcondor.Collector()
coll_ad = coll.locate(htcondor.DaemonTypes.Collector)
rparam = htcondor.RemoteParam(coll_ad)
self.assertTrue("FOO" not in rparam)
rparam["FOO"] = "BAR"
htcondor.send_command(coll_ad, htcondor.DaemonCommands.Reconfig)
rparam2 = htcondor.RemoteParam(coll_ad)
self.assertTrue(rparam2.get("FOO"))
self.assertTrue("FOO" in rparam2)
self.assertEquals(rparam2["FOO"], "BAR")
del rparam["FOO"]
rparam2.refresh()
htcondor.send_command(coll_ad, htcondor.DaemonCommands.Reconfig)
self.assertTrue("FOO" not in rparam2)
self.assertTrue(("ENABLE_CHIRP_DELAYED", "true") in rparam2.items())
def testRemoteSetParam(self):
os.environ["_condor_SETTABLE_ATTRS_READ"] = "FOO"
os.environ["_condor_ENABLE_RUNTIME_CONFIG"] = "TRUE"
self.launch_daemons(["COLLECTOR"])
del os.environ["_condor_SETTABLE_ATTRS_READ"]
#htcondor.param["TOOL_DEBUG"] = "D_NETWORK|D_SECURITY"
htcondor.enable_debug()
coll = htcondor.Collector()
coll_ad = coll.locate(htcondor.DaemonTypes.Collector)
rparam = htcondor.RemoteParam(coll_ad)
self.assertTrue("FOO" not in rparam)
rparam["FOO"] = "BAR"
htcondor.send_command(coll_ad, htcondor.DaemonCommands.Reconfig)
rparam2 = htcondor.RemoteParam(coll_ad)
self.assertTrue(rparam2.get("FOO"))
self.assertTrue("FOO" in rparam2)
self.assertEquals(rparam2["FOO"], "BAR")
del rparam["FOO"]
rparam2.refresh()
htcondor.send_command(coll_ad, htcondor.DaemonCommands.Reconfig)
self.assertTrue("FOO" not in rparam2)
self.assertTrue(("ENABLE_CHIRP_DELAYED", "true") in rparam2.items())
while True:
sub = utils.short_sleep_submit()
schedd = htcondor.Schedd()
log("about to get submit transaction")
with schedd.transaction() as txn:
time.sleep(0.1)
log("got submit transaction")
result = sub.queue(txn, 1)
log("submitted and got result", result)
log("exited submit transaction block")
def query_forever():
while True:
schedd = htcondor.Schedd()
log("about to query")
results = schedd.query()
log("query result length: {}".format(len(results)))
def log(*args):
print("{:.6f}".format(time.time()), threading.current_thread().name, *args)
if __name__ == "__main__":
faulthandler.enable(file=sys.stderr, all_threads=True)
num_query_threads = int(sys.argv[1])
htcondor.enable_debug()
run_test(num_query_threads=num_query_threads)
g_expire_time = time.time() + 15*3600
def lpcUserDN(user):
check_caches()
if isinstance(user, classad.ExprTree):
try:
user = user.eval()
except:
return False
return user in g_cache
classad.register(lpcUserDN)
if __name__ == '__main__':
htcondor.param['CMSLPC_USER_CACHE'] = 'test_lpccache.txt.temp'
shutil.copy('test_lpccache.txt', 'test_lpccache.txt.temp')
htcondor.enable_debug()
print "true ==", lpcUserDN("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=bbockelm/CN=659869/CN=Brian Paul Bockelman")
print "true ==", classad.ExprTree('lpcUserDN("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=bbockelm/CN=659869/CN=Brian Paul Bockelman")').eval()
print "false ==", lpcUserDN("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=bbockelm/CN=659869/CN=Brian Paul Bockelman/false")
print "false ==", classad.ExprTree('lpcUserDN("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=bbockelm/CN=659869/CN=Brian Paul Bockelman/false")').eval()
ad = classad.ClassAd({'x509userproxysubject': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=bbockelm/CN=659869/CN=Brian Paul Bockelman', 'foo': classad.ExprTree('lpcUserDN(x509userproxysubject)')})
print "true ==", ad.eval("foo")
htcondor.param['CMSLPC_USER_URL'] = 'http://hcc-briantest.unl.edu/test_lpccache.txt'
g_expire_time = 0
print "true ==", classad.ExprTree('lpcUserDN("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=bbockelm/CN=659869/CN=Brian Paul Bockelman/true")').eval()
def main(
submit_host=None,
username=None,
pool=None,
authorizations=None,
token_dir=None,
modify_config=True,
debug=False,
):
if debug:
htcondor.param["TOOL_DEBUG"] = "D_SECURITY D_FULLDEBUG"
htcondor.enable_debug()
# We need to override existing settings and authenticate via SSL (anonymously)
htcondor.param["SEC_CLIENT_AUTHENTICATION_METHODS"] = "SSL"
if not sys.platform.startswith("win32"):
# on Ubuntu, set to the correct path (the default is for CentOS)
if not Path("/etc/pki/tls/certs/ca-bundle.crt").exists():
htcondor.param[
"AUTH_SSL_CLIENT_CAFILE"] = "/etc/ssl/certs/ca-certificates.crt"
token_dir.mkdir(parents=True, exist_ok=True)
token_dir.chmod(0o700)
htcondor.param["SEC_TOKEN_DIRECTORY"] = str(token_dir)
if modify_config:
config_path = Path.home() / ".condor" / "user_config"
config_path.parent.mkdir(parents=True, exist_ok=True)
config_lines = [
f"SEC_TOKEN_DIRECTORY = {str(token_dir)}",
]
if not config_path.exists() or any(
config_line not in config_path.read_text()
for config_line in config_lines):
with config_path.open(mode="a") as f:
f.writelines(config_lines)
print(f"Added config to {config_path}")
if submit_host is None:
submit_host = input(
"What is the address of your submit host (e.g., submit3.chtc.wisc.edu)? "
)
if username is None:
username = input("What is your username on your submit host? ")
identity = f"{username}@fs"
if pool is None:
pool = input("What is the address of your pool's central manager? ")
if authorizations is None:
authorizations = [
authz.upper() for authz in input(
"What are the authorizations you need (e.g., READ WRITE)? ").
split()
]
collector = htcondor.Collector(pool)
target = collector.locate(htcondor.DaemonTypes.Schedd, submit_host)
# Pretend that the target is a collector
target["MyType"] = "Collector"
print("Requesting token...")
request = make_token_request(identity, authorizations, target)
lines = [
f"Your token request id is: {request.request_id}",
f"To approve the token request, log in to {username}@{submit_host} via SSH and run:",
f"condor_token_request_approve -name {submit_host} -type SCHEDD -reqid {request.request_id}",
]
print("\n".join(lines))
token = request.result(0)
token_name = f"remote-submit-for-{submit_host}"
token.write(token_name)
print(f"Success! Token saved to {token_dir / token_name}")