def execute(self):
"""Execute / submit a job with HTCondor."""
os.chdir(self.workflow_workspace)
job_ad = classad.ClassAd()
job_ad['JobDescription'] = \
self.workflow.get_full_workflow_name() + '_' + self.job_name
job_ad['JobMaxRetries'] = 3
job_ad['OnExitRemove'] = classad.ExprTree(
'NumJobCompletions > JobMaxRetries || ExitCode == 0')
job_ad['DockerImage'] = self.docker_img
job_ad['WantDocker'] = True
job_ad['Cmd'] = './job_wrapper.sh'
job_ad['Arguments'] = self._format_arguments()
job_ad['Environment'] = self._format_env_vars()
job_ad['Out'] = classad.ExprTree(
'strcat("reana_job.", ClusterId, ".", ProcId, ".out")')
job_ad['Err'] = classad.ExprTree(
'strcat("reana_job.", ClusterId, ".", ProcId, ".err")')
job_ad['log'] = classad.ExprTree(
'strcat("reana_job.", ClusterId, ".err")')
job_ad['ShouldTransferFiles'] = 'YES'
job_ad['WhenToTransferOutput'] = 'ON_EXIT'
job_ad['TransferInput'] = self._get_input_files()
job_ad['TransferOutput'] = '.'
job_ad['PeriodicRelease'] = classad.ExprTree('(HoldReasonCode == 35)')
job_ad['MaxRunTime'] = 3600
clusterid = self._submit(job_ad)
logging.warning("Submitting job clusterid: {0}".format(clusterid))
return clusterid
def import_condor_info():
try:
redis_con = setup_redis_connection()
condor_resources = redis_con.get("condor-resources")
condor_jobs = redis_con.get("condor-jobs")
condor_resources = json.loads(condor_resources)
condor_jobs = json.loads(condor_jobs)
for job in condor_jobs:
# expression trees must be cast as a string to make them json serializable
# we must rebuild the tree from the string on this side.
req_etree = classad.ExprTree(str(job["Requirements"]))
job["Requirements"] = req_etree
for resource in condor_resources:
if "Start" in resource:
start_etree = classad.ExprTree(str(resource["Start"]))
resource["Start"] = start_etree
return condor_resources, condor_jobs
except Exception as e:
print(e)
print("Exiting due to exception")
return None
def makePrioCorrectionsAds():
"""
Optimize the PostJobPrio* entries for HTCondor matchmaking.
This will sort jobs within the schedd along the following criteria (higher is better):
1) Workflow ID (lower is better).
2) Step in workflow (later is better)
3) # of sites in whitelist (lower is better).
4) Estimated job runtime (lower is better).
5) Estimated job disk requirements (lower is better).
"""
anAd = classad.ClassAd()
anAd["GridResource"] = "condor localhost localhost"
anAd["TargetUniverse"] = 5
anAd["Name"] = "Prio Corrections"
anAd["Requirements"] = classad.ExprTree(
"(target.HasPrioCorrection isnt true)")
anAd["set_HasPrioCorrection"] = True
anAd["set_HasBeenRouted"] = False
# -1 * Number of sites in workflow.
anAd["copy_PostJobPrio1"] = "WMAgent_PostJobPrio1"
# -1 * Workflow ID (newer workflows have higher numbers)
anAd["copy_PostJobPrio2"] = "WMAgent_PostJobPrio2"
anAd["eval_set_JR_PostJobPrio1"] = classad.ExprTree(
"WMAgent_PostJobPrio2*100*1000 + size(WMAgent_SubTaskName)*100 + WMAgent_PostJobPrio1"
)
anAd["eval_set_JR_PostJobPrio2"] = classad.ExprTree(
"-MaxWallTimeMins - RequestDisk/1000000")
anAd["set_PostJobPrio1"] = classad.Attribute("JR_PostJobPrio1")
anAd["set_PostJobPrio2"] = classad.Attribute("JR_PostJobPrio2")
print anAd
def makePerformanceCorrectionsAds(configs):
for memory in configs['memory']:
wfs = configs['memory'][memory]
anAd = classad.ClassAd()
anAd["GridResource"] = "condor localhost localhost"
anAd["TargetUniverse"] = 5
anAd["Name"] = str("Set memory requirement to %s" % memory)
anAd["MemoryTasknames"] = map(str, wfs)
memory_names_escaped = anAd.lookup('MemoryTasknames').__repr__()
exp = classad.ExprTree(
'member(target.WMAgent_SubTaskName, %s) && (target.HasBeenMemoryTuned =!= true)'
% (memory_names_escaped))
anAd["Requirements"] = classad.ExprTree(str(exp))
anAd['set_HasBeenMemoryTuned'] = True
anAd['set_HasBeenRouted'] = False
anAd['set_RequestMemory'] = int(memory)
print anAd
for timing in configs['time']:
wfs = configs['time'][timing]
anAd = classad.ClassAd()
anAd["GridResource"] = "condor localhost localhost"
anAd["TargetUniverse"] = 5
anAd["Name"] = str("Set timing requirement to %s" % timing)
anAd["TimeTasknames"] = map(str, wfs)
time_names_escaped = anAd.lookup('TimeTasknames').__repr__()
exp = classad.ExprTree(
'member(target.WMAgent_SubTaskName, %s) && (target.HasBeenTimingTuned =!= true)'
% (time_names_escaped))
anAd["Requirements"] = classad.ExprTree(str(exp))
anAd['set_HasBeenTimingTuned'] = True
anAd['set_HasBeenRouted'] = False
anAd['set_MaxWallTimeMins'] = int(timing)
print anAd
def test_temp_scope(self):
expr = classad.ExprTree("foo")
self.assertEquals("bar", expr.eval({"foo": "bar"}))
ad = classad.ClassAd({"foo": "baz", "test": classad.ExprTree("foo")})
expr = ad["test"]
self.assertEquals("baz", expr.eval())
self.assertEquals("bar", expr.eval({"foo": "bar"}))
self.assertEquals("bar", expr.eval({"foo": "bar"}))
self.assertEquals("baz", expr.eval())
def submitDirect(self, schedd, cmd, arg, info): #pylint: disable=R0201
"""
Submit directly to the schedd using the HTCondor module
"""
dagAd = classad.ClassAd()
addCRABInfoToClassAd(dagAd, info)
# NOTE: Changes here must be synchronized with the job_submit in DagmanCreator.py in CAFTaskWorker
dagAd["CRAB_Attempt"] = 0
dagAd["JobUniverse"] = 12
dagAd["HoldKillSig"] = "SIGUSR1"
dagAd["Out"] = os.path.join(info['scratch'], "request.out")
dagAd["Err"] = os.path.join(info['scratch'], "request.err")
dagAd["Cmd"] = cmd
dagAd['Args'] = arg
dagAd["TransferInput"] = info['inputFilesString']
dagAd["LeaveJobInQueue"] = classad.ExprTree("(JobStatus == 4) && ((StageOutFinish =?= UNDEFINED) || (StageOutFinish == 0))")
dagAd["TransferOutput"] = info['outputFilesString']
dagAd["OnExitRemove"] = classad.ExprTree("( ExitSignal =?= 11 || (ExitCode =!= UNDEFINED && ExitCode >=0 && ExitCode <= 2))")
dagAd["OtherJobRemoveRequirements"] = classad.ExprTree("DAGManJobId =?= ClusterId")
dagAd["RemoveKillSig"] = "SIGUSR1"
dagAd["Environment"] = classad.ExprTree('strcat("PATH=/usr/bin:/bin CONDOR_ID=", ClusterId, ".", ProcId)')
dagAd["RemoteCondorSetup"] = info['remote_condor_setup']
dagAd["Requirements"] = classad.ExprTree('true || false')
dagAd["TaskType"] = "ROOT"
dagAd["X509UserProxy"] = info['userproxy']
r, w = os.pipe()
rpipe = os.fdopen(r, 'r')
wpipe = os.fdopen(w, 'w')
if os.fork() == 0:
#pylint: disable=W0212
try:
rpipe.close()
try:
resultAds = []
htcondor.SecMan().invalidateAllSessions()
os.environ['X509_USER_PROXY'] = info['userproxy']
schedd.submit(dagAd, 1, True, resultAds)
schedd.spool(resultAds)
wpipe.write("OK")
wpipe.close()
os._exit(0)
except Exception: #pylint: disable=W0703
wpipe.write(str(traceback.format_exc()))
finally:
os._exit(1)
wpipe.close()
results = rpipe.read()
if results != "OK":
raise Exception("Failure when submitting HTCondor task: %s" % results)
schedd.reschedule()
def makeSortAds():
anAd = classad.ClassAd()
anAd["GridResource"] = "condor localhost localhost"
anAd["TargetUniverse"] = 5
anAd["Name"] = "Sort Ads"
anAd["Requirements"] = classad.ExprTree(
"(sortStringSet(\"\") isnt error) && (target.HasBeenRouted is false) && (target.HasBeenSorted isnt true)"
)
anAd["copy_DESIRED_Sites"] = "Prev_DESIRED_Sites"
anAd["eval_set_DESIRED_Sites"] = classad.ExprTree(
"debug(sortStringSet(Prev_DESIRED_Sites))")
anAd["set_HasBeenSorted"] = True
anAd['set_HasBeenRouted'] = False
def updateSiteInformation(self, jobs, siteName, excludeSite):
"""
_updateSiteInformation_
Modify condor classAd for all Idle jobs for a site if it has gone Down, Draining or Aborted.
Kill all jobs if the site is the only site for the job.
This expects: excludeSite = False when moving to Normal
excludeSite = True when moving to Down, Draining or Aborted
"""
jobInfo, sd = self.getClassAds()
jobtokill = []
for job in jobs:
jobID = job['id']
jobAd = jobInfo.get(jobID)
if not jobAd:
logging.debug("No jobAd received for jobID %i" % jobID)
else:
desiredSites = jobAd.get('DESIRED_Sites').split(', ')
extDesiredSites = jobAd.get('ExtDESIRED_Sites').split(', ')
if excludeSite:
#Remove siteName from DESIRED_Sites if job has it
if siteName in desiredSites and siteName in extDesiredSites:
usi = desiredSites
if len(usi) > 1:
usi.remove(siteName)
usi = ','.join(map(str, usi))
sd.edit('WMAgent_JobID == %i' % jobID,
"DESIRED_Sites",
classad.ExprTree('"%s"' % usi))
else:
jobtokill.append(job)
else:
#If job doesn't have the siteName in the siteList, just ignore it
logging.debug(
"Cannot find siteName %s in the sitelist" %
siteName)
else:
#Add siteName to DESIRED_Sites if ExtDESIRED_Sites has it (moving back to Normal)
if siteName not in desiredSites and siteName in extDesiredSites:
usi = desiredSites
usi.append(siteName)
usi = ','.join(map(str, usi))
sd.edit('WMAgent_JobID == %i' % jobID, "DESIRED_Sites",
classad.ExprTree('"%s"' % usi))
else:
#If job doesn't have the siteName in the siteList, just ignore it
logging.debug(
"Cannot find siteName %s in the sitelist" %
siteName)
return jobtokill
def addCRABInfoToClassAd(ad, info):
"""
Given a submit ClassAd, add in the appropriate CRAB_* attributes
from the info directory
"""
for adName, dictName in SUBMIT_INFO:
ad[adName] = classad.ExprTree(str(info[dictName]))
def test_ad_refs(self):
ad = classad.ClassAd()
ad["foo"] = classad.ExprTree("bar + baz")
ad["bar"] = 2.1
ad["baz"] = 4
self.assertEqual(ad["foo"].__repr__(), "bar + baz")
self.assertEqual(ad.eval("foo"), 6.1)
def test_operator(self):
expr = classad.Literal(1) + 2
self.assertTrue(isinstance(expr, classad.ExprTree))
self.assertTrue(expr)
self.assertTrue(expr.sameAs(classad.ExprTree('1 + 2')))
expr = classad.Literal(1) & 2
self.assertTrue(isinstance(expr, classad.ExprTree))
self.assertEquals(expr.eval(), 0)
self.assertTrue(expr.sameAs(classad.ExprTree('1 & 2')))
expr = classad.Attribute("foo").is_(classad.Value.Undefined)
self.assertTrue(expr.eval())
ad = classad.ClassAd("[foo = 1]")
expr = classad.Attribute("foo").isnt_(classad.Value.Undefined)
self.assertTrue(expr.eval(ad))
expr = classad.Literal(1).and_(classad.Literal(2))
self.assertRaises(RuntimeError, bool, expr)
def makeResizeAds(config):
policies = {}
for workflow, info in config.get('resizing', {}).items():
minCores = info.get("minCores", 3)
maxCores = info.get("maxCores", 8)
memoryPerThread = info.get("memoryPerThread")
workflows = policies.setdefault((minCores, maxCores, memoryPerThread),
set())
workflows.add(workflow)
for policy, workflows in policies.items():
minCores, maxCores, memoryPerThread = policy
anAd = classad.ClassAd()
anAd['GridResource'] = 'condor localhost localhost'
anAd['TargetUniverse'] = 5
# Same trick as above to convert the set to a ClassAd list.
anAd["OverflowTasknames"] = map(str, workflows)
tasks_escaped = anAd.lookup('OverflowTasknames').__repr__()
del anAd['OverflowTaskNames']
anAd['Name'] = 'Resize Jobs (%d-%d cores, %d MB/thread)' % (
minCores, maxCores, memoryPerThread)
anAd['Requirements'] = classad.ExprTree(
'(target.WMCore_ResizeJob is False) && member(target.WMAgent_SubTaskName, %s)'
% tasks_escaped)
anAd['set_WMCore_ResizeJob'] = True
anAd['set_MinCores'] = minCores
anAd['set_MaxCores'] = maxCores
anAd['set_HasBeenRouted'] = False
anAd['set_ExtraMemory'] = memoryPerThread
#anAd['set_RequestMemory'] = classad.ExprTree('OriginalMemory + %d * ( WMCore_ResizeJob ? ( RequestCpus - OriginalCpus ) : 0 )' % memoryPerThread)
print anAd
def get_schedds(args=None, collectors=None):
"""
Return a list of schedd ads representing all the schedds in the pool.
"""
collectors = collectors or [
"cmssrv221.fnal.gov:9620", "cmsgwms-collector-tier0.cern.ch:9620",
"cmssrv276.fnal.gov"
]
schedd_query = classad.ExprTree('!isUndefined(CMSGWMS_Type)')
schedd_ads = {}
for host in collectors:
coll = htcondor.Collector(host)
try:
schedds = coll.query(
htcondor.AdTypes.Schedd,
schedd_query,
projection=["MyAddress", "ScheddIpAddr", "Name"])
except IOError, e:
logging.warning(str(e))
continue
for schedd in schedds:
try:
schedd_ads[schedd['Name']] = schedd
except KeyError:
pass
def _combine(
self, other: Union["ConstraintHandle", classad.ExprTree, str], combinator
):
if isinstance(other, ConstraintHandle) and (
self.collector != other.collector or self.scheduler != other.scheduler
):
raise exceptions.InvalidHandle(
"Cannot construct a handle for separate schedds"
)
if isinstance(other, ConstraintHandle):
c = other.constraint
elif isinstance(other, classad.ExprTree):
c = other
elif isinstance(other, str):
c = classad.ExprTree(other)
else:
raise exceptions.InvalidHandle(
f"Cannot construct a combined handle from {self} and {other} because it is not a ConstraintHandle, ExprTree, or cannot be parsed into an ExprTree"
)
return ConstraintHandle(
combinator(self.constraint, c),
collector=self.collector,
scheduler=self.scheduler,
)
def get_schedds():
#schedd_query = classad.ExprTree('CMSGWMS_Type =?="prodschedd" && Name =!= "vocms001.cern.ch" && Name =!= "vocms047.cern.ch" && Name=!="vocms015.cern.ch"')
schedd_query = classad.ExprTree('!isUndefined(CMSGWMS_Type)')
coll = htcondor.Collector("cmssrv221.fnal.gov:9620")
schedd_ads = coll.query(htcondor.AdTypes.Schedd, schedd_query, projection=["MyAddress", "ScheddIpAddr", "Name"])
random.shuffle(schedd_ads)
return schedd_ads
def test_ad_special_values(self):
ad = classad.ClassAd()
ad["foo"] = classad.ExprTree('regexp(12, 34)')
ad["bar"] = classad.Value.Undefined
self.assertEqual(ad["foo"].eval(), classad.Value.Error)
self.assertNotEqual(ad["foo"].eval(), ad["bar"])
self.assertEqual(classad.Value.Undefined, ad["bar"])
def test_register(self):
class BadException(Exception):
pass
def myAdd(a, b):
return a + b
def myBad(a, b):
raise BadException("bad")
def myComplex(a):
return 1j # ClassAds have no complex numbers, not able to convert from python to an expression
def myExpr(**kw):
return classad.ExprTree(
"foo"
) # Functions must return values; this becomes "undefined".
def myFoo(foo):
return foo['foo']
def myIntersect(a, b):
return set(a).intersection(set(b))
classad.register(myAdd)
classad.register(myAdd, name='myAdd2')
classad.register(myBad)
classad.register(myComplex)
classad.register(myExpr)
classad.register(myFoo)
classad.register(myIntersect)
self.assertEqual(3, classad.ExprTree('myAdd(1, 2)').eval())
self.assertEqual(3, classad.ExprTree('myAdd2(1, 2)').eval())
self.assertRaises(BadException, classad.ExprTree('myBad(1, 2)').eval)
self.assertRaises(TypeError, classad.ExprTree('myComplex(1)').eval)
self.assertEqual(classad.Value.Undefined,
classad.ExprTree('myExpr()').eval())
self.assertEqual(
classad.ExprTree('myExpr()').eval(classad.ClassAd({"foo": 2})), 2)
self.assertRaises(TypeError,
classad.ExprTree('myAdd(1)').eval
) # myAdd requires 2 arguments; only one is given.
self.assertEqual(classad.ExprTree('myFoo([foo = 1])').eval(), 1)
self.assertEqual(
classad.ExprTree('size(myIntersect({1, 2}, {2, 3}))').eval(), 1)
self.assertEqual(
classad.ExprTree('myIntersect({1, 2}, {2, 3})[0]').eval(), 2)
def test_proc_expr(self):
"""get_num_procs() should be able to handle attributes set to ClassAd expressions
"""
for attr in condor_meter.PROC_ATTRS:
jobad = classad.ClassAd()
jobad[attr] = classad.ExprTree('2 + 2')
procs = condor_meter.get_num_procs(jobad)
self.assertEquals(procs, 4)
def test_state(self):
def myFunc(state):
return 1 if state else 0
classad.register(myFunc)
self.assertEqual(0, classad.ExprTree('myFunc(false)').eval())
self.assertEqual(1, classad.ExprTree('myFunc("foo")').eval())
ad = classad.ClassAd("""[foo = myFunc(); bar = 2]""")
self.assertEqual(1, ad.eval('foo'))
ad['foo'] = classad.ExprTree('myFunc(1)')
self.assertRaises(TypeError, ad.eval, ('foo', ))
def myFunc(arg1, **kw):
return kw['state']['bar']
classad.register(myFunc)
self.assertEqual(2, ad.eval('foo'))
def execute(self):
"""Execute / submit a job with HTCondor."""
os.chdir(self.workflow_workspace)
job_ad = classad.ClassAd()
job_ad["JobDescription"] = (
self.workflow.get_full_workflow_name() + "_" + self.job_name
)
job_ad["JobMaxRetries"] = 3
job_ad["LeaveJobInQueue"] = classad.ExprTree(
"(JobStatus == 4) && ((StageOutFinish =?= UNDEFINED) || "
"(StageOutFinish == 0))"
)
job_ad["Cmd"] = (
"./job_wrapper.sh"
if not self.unpacked_img
else "./job_singularity_wrapper.sh"
)
if not self.unpacked_img:
job_ad["Arguments"] = self._format_arguments()
job_ad["DockerImage"] = self.docker_img
job_ad["WantDocker"] = True
job_ad["Environment"] = self._format_env_vars()
job_ad["Out"] = classad.ExprTree(
'strcat("reana_job.", ClusterId, ".", ProcId, ".out")'
)
job_ad["Err"] = classad.ExprTree(
'strcat("reana_job.", ClusterId, ".", ProcId, ".err")'
)
job_ad["log"] = classad.ExprTree('strcat("reana_job.", ClusterId, ".err")')
job_ad["ShouldTransferFiles"] = "YES"
job_ad["WhenToTransferOutput"] = "ON_EXIT"
job_ad["TransferInput"] = self._get_input_files()
job_ad["TransferOutput"] = "."
job_ad["PeriodicRelease"] = classad.ExprTree("(HoldReasonCode == 35)")
if self.htcondor_max_runtime in HTCONDOR_JOB_FLAVOURS.keys():
job_ad["JobFlavour"] = self.htcondor_max_runtime
elif str.isdigit(self.htcondor_max_runtime):
job_ad["MaxRunTime"] = int(self.htcondor_max_runtime)
else:
job_ad["MaxRunTime"] = 3600
if self.htcondor_accounting_group:
job_ad["AccountingGroup"] = self.htcondor_accounting_group
future = current_app.htcondor_executor.submit(self._submit, job_ad)
clusterid = future.result()
return clusterid
def test_register_custom_function():
def concatenateLists(list1, list2):
return list1 + list2
classad.register(concatenateLists)
assert classad.ExprTree("concatenateLists({1, 2}, {3, 4})").eval() == [
1, 2, 3, 4
]
def test_custom_function_can_see_python_values():
local = 1
def add(a):
return a + local
classad.register(add)
assert classad.ExprTree("add(1)").eval() == 2
def evalExpressionStr(expression_str, context):
"""Evaluate a classad expression (in a string) in the given context (a
ClassAd).
Can raise:
- SyntaxError (if expression_str is unparseable)
"""
return context.flatten(classad.ExprTree(expression_str))
def test_pickle(self):
ad = classad.ClassAd({"one": 1})
expr = classad.ExprTree("2+2")
pad = pickle.dumps(ad)
pexpr = pickle.dumps(expr)
ad2 = pickle.loads(pad)
expr2 = pickle.loads(pexpr)
self.assertEquals(ad2.__repr__(), "[ one = 1 ]")
self.assertEquals(expr2.__repr__(), "2 + 2")
def getClusterAd():
""" Mimic the same method of SimpleCondorPlugin """
ad = classad.ClassAd()
# ad['universe'] = "vanilla"
ad['ShouldTransferFiles'] = "YES"
ad['WhenToTransferOutput'] = "ON_EXIT"
ad['UserLogUseXML'] = True
ad['JobNotification'] = 0
ad['Cmd'] = TEST_DIR + 'submit_fake.sh'
# Investigate whether we should pass the absolute path for Out and Err ads,
# just as we did for UserLog. There may be issues, more info on WMCore #7362
ad['Out'] = classad.ExprTree(
'strcat("condor.", ClusterId, ".", ProcId, ".out")')
ad['Err'] = classad.ExprTree(
'strcat("condor.", ClusterId, ".", ProcId, ".err")')
ad['UserLog'] = classad.ExprTree(
'strcat(Iwd, "/condor.", ClusterId, ".", ProcId, ".log")')
ad['WMAgent_AgentName'] = 'WMAgentCommissioning'
ad['JobLeaseDuration'] = classad.ExprTree(
'isUndefined(MachineAttrMaxHibernateTime0) ? 1200 : MachineAttrMaxHibernateTime0'
)
ad['PeriodicRemove'] = classad.ExprTree(
'( JobStatus =?= 5 ) && ( time() - EnteredCurrentStatus > 10 * 60 )')
removeReasonExpr = 'PeriodicRemove ? "Job automatically removed for being in Held status" : ""'
ad['PeriodicRemoveReason'] = classad.ExprTree(removeReasonExpr)
# Required for global pool accounting
ad['AcctGroup'] = 'production'
ad['AcctGroupUser'] = '******'
ad['AccountingGroup'] = "%s.%s" % (ad['AcctGroup'], ad['AcctGroupUser'])
# Customized classAds for this plugin
ad['DESIRED_Archs'] = "INTEL,X86_64"
ad['Rank'] = 0.0
ad['TransferIn'] = False
ad['JobMachineAttrs'] = "GLIDEIN_CMSSite"
ad['JobAdInformationAttrs'] = (
"JobStatus,QDate,EnteredCurrentStatus,JobStartDate,DESIRED_Sites,"
"ExtDESIRED_Sites,WMAgent_JobID,MachineAttrGLIDEIN_CMSSite0")
# TODO: remove when 8.5.7 is deployed
paramsToAdd = htcondor.param['SUBMIT_ATTRS'].split(
) + htcondor.param['SUBMIT_EXPRS'].split()
paramsToSkip = [
'accounting_group', 'use_x509userproxy', 'PostJobPrio2',
'JobAdInformationAttrs'
]
for param in paramsToAdd:
if (param not in ad) and (param
in htcondor.param) and (param
not in paramsToSkip):
ad[param] = classad.ExprTree(htcondor.param[param])
return ad
def makePerformanceCorrectionsAds(configs):
m_config = configs.get('memory', {})
for memory in m_config:
wfs = m_config[memory]
anAd = classad.ClassAd()
anAd["GridResource"] = "condor localhost localhost"
anAd["TargetUniverse"] = 5
anAd["Name"] = str("Set memory requirement to %s" % memory)
anAd["MemoryTasknames"] = map(str, wfs)
memory_names_escaped = anAd.lookup('MemoryTasknames').__repr__()
exp = classad.ExprTree(
'member(target.WMAgent_SubTaskName, %s) && ((target.HasBeenMemoryTuned =!= true) || (target.OriginalMemory =!= %d))'
% (memory_names_escaped,
int(memory))) ## just set to a different value
anAd["Requirements"] = classad.ExprTree(str(exp))
anAd['set_HasBeenMemoryTuned'] = True
anAd['set_HasBeenRouted'] = False
anAd['set_OriginalMemory'] = int(memory)
print anAd
t_config = configs.get('time', {})
for timing in t_config:
wfs = t_config[timing]
anAd = classad.ClassAd()
anAd["GridResource"] = "condor localhost localhost"
anAd["TargetUniverse"] = 5
anAd["Name"] = str("Set timing requirement to %s" % timing)
anAd["TimeTasknames"] = map(str, wfs)
time_names_escaped = anAd.lookup('TimeTasknames').__repr__()
exp = classad.ExprTree(
'member(target.WMAgent_SubTaskName, %s) && ((target.HasBeenTimingTuned =!= true) || (target.EstimatedSingleCoreMins <= %d))'
% (time_names_escaped, int(timing)))
anAd["Requirements"] = classad.ExprTree(str(exp))
anAd['set_HasBeenTimingTuned'] = True
anAd['set_HasBeenRouted'] = False
anAd['set_EstimatedSingleCoreMins'] = int(timing)
anAd['set_OriginalMaxWallTimeMins'] = classad.ExprTree(
'EstimatedSingleCoreMins / OriginalCpus')
print anAd
s_config = configs.get('slope', {})
for slope in s_config:
wfs = s_config[slope]
anAd = classad.ClassAd()
anAd["GridResource"] = "condor localhost localhost"
anAd["TargetUniverse"] = 5
anAd["Name"] = str("Set memory per thread requirement to %s" % slope)
anAd["TimeTasknames"] = map(str, wfs)
time_names_escaped = anAd.lookup('TimeTasknames').__repr__()
exp = classad.ExprTree(
'member(target.WMAgent_SubTaskName, %s) && (target.ExtraMemory =!= %d)'
%
(time_names_escaped, int(slope))) ## just set to a different value
anAd["Requirements"] = classad.ExprTree(str(exp))
anAd['set_HasBeenSlopeTuned'] = True
anAd['set_HasBeenRouted'] = False
anAd['set_ExtraMemory'] = int(slope)
print anAd
def test_register_with_exception():
def bad(a, b):
raise Exception("oops")
classad.register(bad)
with pytest.raises(Exception) as e:
classad.ExprTree("bad(1, 2)").eval()
assert str(e.value) == "oops"
def makeReadAds(config):
for needs, tasks in config.get('read', {}).items():
anAd = classad.ClassAd()
set_read = int(float(needs))
anAd["Name"] = str("Set read requirement to %s" % set_read)
anAd["GridResource"] = "condor localhost localhost"
anAd["TargetUniverse"] = 5
anAd["JobRouterTasknames"] = map(str, tasks)
task_names_escaped = anAd.lookup('JobRouterTasknames').__repr__()
del anAd["JobRouterTasknames"]
exp = classad.ExprTree(
'member(target.WMAgent_SubTaskName, %s) && (EstimatedInputRateKBs =!= %d)'
% (task_names_escaped,
int(set_read))) ## just set to a different value
anAd["Requirements"] = classad.ExprTree(str(exp))
anAd["set_HasBeenRouted"] = False
anAd["set_HasBeenReadTuned"] = True
anAd["set_EstimatedInputRateKBs"] = int(set_read)
print anAd
def __when_preemptible__():
try:
return int(
classad.ExprTree(
htcondor.param.get('IDLE_WARNING_TRIGGER')).eval())
except Exception as e:
log.critical("""Could not find or evaluate IDLE_WARNING_TRIGGER in
HTCondor configuration. Returning None. Exception:
{0}""".format(e))
return None
def test_list_conversion(self):
ad = dict(classad.ClassAd("[a = {1,2,3}]"))
self.assertTrue(isinstance(ad["a"], types.ListType))
self.assertTrue(isinstance(ad["a"][0], types.LongType))
def listAdd(a, b):
return a + b
classad.register(listAdd)
self.assertEqual(classad.ExprTree("listAdd({1,2}, {3,4})")[0], 1)
