def pytest_report_header(config, startdir):
return [
"",
"Base per-test directory: {}".format(get_base_test_dir(config)),
"Platform:\n{}".format(platform.platform()),
"Python version:\n{}".format(sys.version),
"Python bindings version:\n{}".format(htcondor.version()),
"HTCondor version:\n{}".format(
subprocess.run(
["condor_version"],
stdout=subprocess.PIPE).stdout.decode("utf-8").strip()),
"",
]
def get_condor_version():
"""Should only be called on a submit machine
"""
info = None
try:
import htcondor
info = htcondor.version()
except ImportError:
assert_command_exists('condor_version')
proc = subprocess.Popen(['condor_version'],
stdout=subprocess.PIPE,
shell=True)
out, err = proc.communicate()
info = out
finally:
if info is None:
raise OSError('Could not find HTCondor version.')
condor_version = parse_condor_version(info)
return condor_version
def get_condor_version():
"""Should only be called on a submit machine
"""
condor_info_str = None
try:
import htcondor
condor_info_str = htcondor.version()
except ImportError:
assert_command_exists('condor_version')
proc = subprocess.Popen(['condor_version'],
stdout=subprocess.PIPE,
shell=True)
out, err = proc.communicate()
condor_info_str = out
finally:
if condor_info_str is None:
raise OSError('Could not find HTCondor version.')
condor_version_str = re.search(r'CondorVersion: \s*([\d.]+)',
condor_info_str).group(1)
condor_version = tuple(map(int, condor_version_str.split('.')))
return condor_version
def condor_version():
"""
output is like
'$CondorVersion: 8.6.12 Jul 31 2018 BuildID: 446077 $'
"""
return htcondor.version()
def submit_DAG2AD(self, dagfile):
DAGMAN="/usr/bin/condor_dagman"
dag = dagfile
schedd = htcondor.Schedd()
ad = classad.ClassAd({ "JobUniverse": 7, "Cmd": DAGMAN, "Arguments": "-f -l . -Lockfile %s.lock -AutoRescue 1 -DoRescueFrom 0 " \
"-Dag %s -Suppress_notification -CsdVersion '%s' -Force -Dagman %s" % (dag, dag, htcondor.version(), DAGMAN),
"Env": "_CONDOR_MAX_DAGMAN_LOG=0;_CONDOR_DAGMAN_LOG=%s.dagman.out;" \
"_CONDOR_SCHEDD_DAEMON_AD_FILE=%s;_CONDOR_SCHEDD_ADDRESS_FILE=%s" % (dag, htcondor.param["SCHEDD_DAEMON_AD_FILE"], htcondor.param["SCHEDD_ADDRESS_FILE"]),
"EnvDelim": ";",
"Out": "%s.lib.out" % dag,
"Err": "%s.lib.err" % dag,
"ShouldTransferFiles": "IF_NEEDED",
"UserLog": os.path.abspath("%s.dagman.log" % dag),
"KillSig": "SIGTERM",
"RemoveKillSig": "SIGUSR1",
#"OtherJobRemoveRequirements": classad.ExprTree('eval(strcat("DAGManJobId == ", ClusterId))'),
"OnExitRemove": classad.ExprTree('( ExitSignal =?= 11 || ( ExitCode =!= undefined && ExitCode >= 0 && ExitCode <= 2 ) )'),
"FileSystemDomain": htcondor.param['FILESYSTEM_DOMAIN'],
"Requirements": classad.ExprTree('true || false'),
})
cluster = schedd.submit(ad)
print("Submitted as cluster %d" % cluster)
'gocservice': 'org.opensciencegrid.htcondorce',
'idlecores': idle_cores,
'ceunique': '%s/%s-condor' % (htcondor.param['COLLECTOR_HOST'], schedd_name),
'statusinfo': cestatusinfo,
'cestatus': status,
'status': cestatus,
'sitename': sitename,
'batch': batch,
'hostname': htcondor.param['FULL_HOSTNAME'],
'totalcores': total_cores,
'busycores': busy_cores,
'runningjobs': sum(running_vo_jobs.values()),
'idlejobs': sum(idle_vo_jobs.values()),
'totaljobs': sum(total_vo_jobs.values()),
'totalinstances': total_instances,
'condorversion': htcondor.version(),
'ceversion': ceversion,
'cores': cores,
'hepspec_info': hepspec_info,
'endpointtrustedcas': trustedcas,
'dateandtime': dateandtime,
'startdate': scheddstarttime
}
if vonames:
info['acbr'] = '\n'.join(['GlueCEAccessControlBaseRule: VO:%s' % vo for vo in vonames])
print service_template % info
print resource_template % info
print endpoint_template % info
print manager_template % info
print benchmark_template % info
for vo in vonames:
import logging
import time
import os
import util
from distutils.version import StrictVersion
class JobException(Exception):
pass
# Transitions that indicate the job is not running anymore
TERMINAL_TRANS = (2, 5, 9, 12)
VERSION = htcondor.version().split()[1]
logging.basicConfig(
format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
level=logging.DEBUG)
log = logging.getLogger(__name__)
log.info("Iniit v %s", VERSION)
class Job(object):
def __init__(self, job=None, ad=None):
if job and ad:
log.error("Must specify a job with either ad or job, no more")
raise JobException("Invalid Job")
def version():
"""Print HTMap and HTCondor Python bindings version information."""
click.echo(htmap.version())
click.echo(htcondor.version())
def submit_parent_dag(working_dir, source_dir, dest_dir, test_mode=False):
try:
os.makedirs(os.path.join(working_dir, "calc_work"))
except OSError as oe:
if oe.errno != errno.EEXIST:
raise
info = os.path.split(sys.argv[0])
full_exec_path = os.path.join(os.path.abspath(info[0]), info[1])
with open(os.path.join(working_dir, "xfer.dag"), "w") as fd:
fd.write(
PARENT_DAG.format(exec_py=full_exec_path,
source_prefix=source_dir,
dest_prefix=dest_dir,
other_args="--test-mode" if test_mode else "",
transfer_manifest=os.path.join(
dest_dir, "transfer_manifest.txt")))
with open(os.path.join(working_dir, "calc_work", "calc_work.sub"),
"w") as fd:
fd.write(
CALC_WORK_JOB.format(
exec_py=full_exec_path,
source_dir=source_dir,
other_args="--test-mode" if test_mode else ""))
dagman = search_path("condor_dagman")
if not dagman:
print("Unable to find the `condor_dagman` executable in the $PATH")
sys.exit(1)
submit_dict = {
b"universe":
b"scheduler",
b"executable":
dagman.encode('utf-8'),
b"output":
b"xfer.dag.lib.out",
b"error":
b"xfer.dag.lib.err",
b"log":
b"xfer.dag.dagman.log",
b"remove_kill_Sig":
b"SIGUSR1",
b"+OtherJobRemoveRequirements":
b'"DAGManJobId =?= $(cluster)"',
b"on_exit_remove":
b'(ExitSignal =?= 11 || (ExitCode =!= UNDEFINED && '
b'ExitCode >=0 && ExitCode <= 2))',
b"arguments":
b"\"-p 0 -f -l . -Lockfile xfer.dag.lock -AutoRescue 1 -DoRescueFrom 0"
b" -Dag xfer.dag -Suppress_notification -Dagman /usr/bin/condor_dagman"
b" -CsdVersion {}\"".format(htcondor.version().replace(" ", "' '")),
b"environment":
b"_CONDOR_SCHEDD_ADDRESS_FILE={};_CONDOR_MAX_DAGMAN_LOG=0;"
b"_CONDOR_SCHEDD_DAEMON_AD_FILE={};_CONDOR_DAGMAN_LOG=xfer.dag.dagman.out"
.format(htcondor.param[b'SCHEDD_ADDRESS_FILE'],
htcondor.param[b'SCHEDD_DAEMON_AD_FILE'])
}
schedd = htcondor.Schedd()
orig_pwd = os.getcwd()
try:
os.chdir(working_dir)
sub = htcondor.Submit(submit_dict)
with schedd.transaction() as txn:
return sub.queue(txn)
finally:
os.chdir(orig_pwd)
logger.addHandler(handler)
VERSION_RE = re.compile(
r'^(\d+) \. (\d+) (\. (\d+))? ([ab](\d+))?$',
re.VERBOSE | re.ASCII,
)
def parse_version(v: str) -> Tuple[int, int, int, str, int]:
match = VERSION_RE.match(v)
if match is None:
raise Exception(f"Could not determine version info from {v}")
(major, minor, micro, prerelease, prerelease_num) = match.group(1, 2, 4, 5, 6)
out = (
int(major),
int(minor),
int(micro or 0),
prerelease[0] if prerelease is not None else None,
int(prerelease_num) if prerelease_num is not None else None,
)
return out
EXTRACT_HTCONDOR_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+)", flags = re.ASCII)
HTCONDOR_VERSION_INFO = parse_version(EXTRACT_HTCONDOR_VERSION_RE.search(htcondor.version()).group(0))
def condor_version():
"""
output is like
'$CondorVersion: 8.6.12 Jul 31 2018 BuildID: 446077 $'
"""
return htcondor.version()
# this file is intended to be invoked using "/path/to/python test_wheel.py"
import htcondor
import classad
# Test the htcondor module
print(htcondor.version())
try:
collector = htcondor.Collector()
ad = collector.query(htcondor.AdTypes.Collector)
except Exception as e:
print(e)
sys.exit(1)
if len(ad) == 0:
print('Did not find any Collector Classads!')
sys.exit(1)
# Test the classad module
try:
ad = classad.ClassAd()
ad['Five'] = 5
ad['TestExpr'] = classad.ExprTree("Five + 5")
except Exception as e:
print(e)
sys.exit(1)
assert ad.eval('TestExpr') == 10
