def execute(self): """Execute / submit a job with HTCondor.""" os.chdir(self.workflow_workspace) job_ad = classad.ClassAd() job_ad['JobDescription'] = \ self.workflow.get_full_workflow_name() + '_' + self.job_name job_ad['JobMaxRetries'] = 3 job_ad['OnExitRemove'] = classad.ExprTree( 'NumJobCompletions > JobMaxRetries || ExitCode == 0') job_ad['DockerImage'] = self.docker_img job_ad['WantDocker'] = True job_ad['Cmd'] = './job_wrapper.sh' job_ad['Arguments'] = self._format_arguments() job_ad['Environment'] = self._format_env_vars() job_ad['Out'] = classad.ExprTree( 'strcat("reana_job.", ClusterId, ".", ProcId, ".out")') job_ad['Err'] = classad.ExprTree( 'strcat("reana_job.", ClusterId, ".", ProcId, ".err")') job_ad['log'] = classad.ExprTree( 'strcat("reana_job.", ClusterId, ".err")') job_ad['ShouldTransferFiles'] = 'YES' job_ad['WhenToTransferOutput'] = 'ON_EXIT' job_ad['TransferInput'] = self._get_input_files() job_ad['TransferOutput'] = '.' job_ad['PeriodicRelease'] = classad.ExprTree('(HoldReasonCode == 35)') job_ad['MaxRunTime'] = 3600 clusterid = self._submit(job_ad) logging.warning("Submitting job clusterid: {0}".format(clusterid)) return clusterid
def import_condor_info(): try: redis_con = setup_redis_connection() condor_resources = redis_con.get("condor-resources") condor_jobs = redis_con.get("condor-jobs") condor_resources = json.loads(condor_resources) condor_jobs = json.loads(condor_jobs) for job in condor_jobs: # expression trees must be cast as a string to make them json serializable # we must rebuild the tree from the string on this side. req_etree = classad.ExprTree(str(job["Requirements"])) job["Requirements"] = req_etree for resource in condor_resources: if "Start" in resource: start_etree = classad.ExprTree(str(resource["Start"])) resource["Start"] = start_etree return condor_resources, condor_jobs except Exception as e: print(e) print("Exiting due to exception") return None
def makePrioCorrectionsAds(): """ Optimize the PostJobPrio* entries for HTCondor matchmaking. This will sort jobs within the schedd along the following criteria (higher is better): 1) Workflow ID (lower is better). 2) Step in workflow (later is better) 3) # of sites in whitelist (lower is better). 4) Estimated job runtime (lower is better). 5) Estimated job disk requirements (lower is better). """ anAd = classad.ClassAd() anAd["GridResource"] = "condor localhost localhost" anAd["TargetUniverse"] = 5 anAd["Name"] = "Prio Corrections" anAd["Requirements"] = classad.ExprTree( "(target.HasPrioCorrection isnt true)") anAd["set_HasPrioCorrection"] = True anAd["set_HasBeenRouted"] = False # -1 * Number of sites in workflow. anAd["copy_PostJobPrio1"] = "WMAgent_PostJobPrio1" # -1 * Workflow ID (newer workflows have higher numbers) anAd["copy_PostJobPrio2"] = "WMAgent_PostJobPrio2" anAd["eval_set_JR_PostJobPrio1"] = classad.ExprTree( "WMAgent_PostJobPrio2*100*1000 + size(WMAgent_SubTaskName)*100 + WMAgent_PostJobPrio1" ) anAd["eval_set_JR_PostJobPrio2"] = classad.ExprTree( "-MaxWallTimeMins - RequestDisk/1000000") anAd["set_PostJobPrio1"] = classad.Attribute("JR_PostJobPrio1") anAd["set_PostJobPrio2"] = classad.Attribute("JR_PostJobPrio2") print anAd
def makePerformanceCorrectionsAds(configs): for memory in configs['memory']: wfs = configs['memory'][memory] anAd = classad.ClassAd() anAd["GridResource"] = "condor localhost localhost" anAd["TargetUniverse"] = 5 anAd["Name"] = str("Set memory requirement to %s" % memory) anAd["MemoryTasknames"] = map(str, wfs) memory_names_escaped = anAd.lookup('MemoryTasknames').__repr__() exp = classad.ExprTree( 'member(target.WMAgent_SubTaskName, %s) && (target.HasBeenMemoryTuned =!= true)' % (memory_names_escaped)) anAd["Requirements"] = classad.ExprTree(str(exp)) anAd['set_HasBeenMemoryTuned'] = True anAd['set_HasBeenRouted'] = False anAd['set_RequestMemory'] = int(memory) print anAd for timing in configs['time']: wfs = configs['time'][timing] anAd = classad.ClassAd() anAd["GridResource"] = "condor localhost localhost" anAd["TargetUniverse"] = 5 anAd["Name"] = str("Set timing requirement to %s" % timing) anAd["TimeTasknames"] = map(str, wfs) time_names_escaped = anAd.lookup('TimeTasknames').__repr__() exp = classad.ExprTree( 'member(target.WMAgent_SubTaskName, %s) && (target.HasBeenTimingTuned =!= true)' % (time_names_escaped)) anAd["Requirements"] = classad.ExprTree(str(exp)) anAd['set_HasBeenTimingTuned'] = True anAd['set_HasBeenRouted'] = False anAd['set_MaxWallTimeMins'] = int(timing) print anAd
def test_temp_scope(self): expr = classad.ExprTree("foo") self.assertEquals("bar", expr.eval({"foo": "bar"})) ad = classad.ClassAd({"foo": "baz", "test": classad.ExprTree("foo")}) expr = ad["test"] self.assertEquals("baz", expr.eval()) self.assertEquals("bar", expr.eval({"foo": "bar"})) self.assertEquals("bar", expr.eval({"foo": "bar"})) self.assertEquals("baz", expr.eval())
def submitDirect(self, schedd, cmd, arg, info): #pylint: disable=R0201 """ Submit directly to the schedd using the HTCondor module """ dagAd = classad.ClassAd() addCRABInfoToClassAd(dagAd, info) # NOTE: Changes here must be synchronized with the job_submit in DagmanCreator.py in CAFTaskWorker dagAd["CRAB_Attempt"] = 0 dagAd["JobUniverse"] = 12 dagAd["HoldKillSig"] = "SIGUSR1" dagAd["Out"] = os.path.join(info['scratch'], "request.out") dagAd["Err"] = os.path.join(info['scratch'], "request.err") dagAd["Cmd"] = cmd dagAd['Args'] = arg dagAd["TransferInput"] = info['inputFilesString'] dagAd["LeaveJobInQueue"] = classad.ExprTree("(JobStatus == 4) && ((StageOutFinish =?= UNDEFINED) || (StageOutFinish == 0))") dagAd["TransferOutput"] = info['outputFilesString'] dagAd["OnExitRemove"] = classad.ExprTree("( ExitSignal =?= 11 || (ExitCode =!= UNDEFINED && ExitCode >=0 && ExitCode <= 2))") dagAd["OtherJobRemoveRequirements"] = classad.ExprTree("DAGManJobId =?= ClusterId") dagAd["RemoveKillSig"] = "SIGUSR1" dagAd["Environment"] = classad.ExprTree('strcat("PATH=/usr/bin:/bin CONDOR_ID=", ClusterId, ".", ProcId)') dagAd["RemoteCondorSetup"] = info['remote_condor_setup'] dagAd["Requirements"] = classad.ExprTree('true || false') dagAd["TaskType"] = "ROOT" dagAd["X509UserProxy"] = info['userproxy'] r, w = os.pipe() rpipe = os.fdopen(r, 'r') wpipe = os.fdopen(w, 'w') if os.fork() == 0: #pylint: disable=W0212 try: rpipe.close() try: resultAds = [] htcondor.SecMan().invalidateAllSessions() os.environ['X509_USER_PROXY'] = info['userproxy'] schedd.submit(dagAd, 1, True, resultAds) schedd.spool(resultAds) wpipe.write("OK") wpipe.close() os._exit(0) except Exception: #pylint: disable=W0703 wpipe.write(str(traceback.format_exc())) finally: os._exit(1) wpipe.close() results = rpipe.read() if results != "OK": raise Exception("Failure when submitting HTCondor task: %s" % results) schedd.reschedule()
def makeSortAds(): anAd = classad.ClassAd() anAd["GridResource"] = "condor localhost localhost" anAd["TargetUniverse"] = 5 anAd["Name"] = "Sort Ads" anAd["Requirements"] = classad.ExprTree( "(sortStringSet(\"\") isnt error) && (target.HasBeenRouted is false) && (target.HasBeenSorted isnt true)" ) anAd["copy_DESIRED_Sites"] = "Prev_DESIRED_Sites" anAd["eval_set_DESIRED_Sites"] = classad.ExprTree( "debug(sortStringSet(Prev_DESIRED_Sites))") anAd["set_HasBeenSorted"] = True anAd['set_HasBeenRouted'] = False
def updateSiteInformation(self, jobs, siteName, excludeSite): """ _updateSiteInformation_ Modify condor classAd for all Idle jobs for a site if it has gone Down, Draining or Aborted. Kill all jobs if the site is the only site for the job. This expects: excludeSite = False when moving to Normal excludeSite = True when moving to Down, Draining or Aborted """ jobInfo, sd = self.getClassAds() jobtokill = [] for job in jobs: jobID = job['id'] jobAd = jobInfo.get(jobID) if not jobAd: logging.debug("No jobAd received for jobID %i" % jobID) else: desiredSites = jobAd.get('DESIRED_Sites').split(', ') extDesiredSites = jobAd.get('ExtDESIRED_Sites').split(', ') if excludeSite: #Remove siteName from DESIRED_Sites if job has it if siteName in desiredSites and siteName in extDesiredSites: usi = desiredSites if len(usi) > 1: usi.remove(siteName) usi = ','.join(map(str, usi)) sd.edit('WMAgent_JobID == %i' % jobID, "DESIRED_Sites", classad.ExprTree('"%s"' % usi)) else: jobtokill.append(job) else: #If job doesn't have the siteName in the siteList, just ignore it logging.debug( "Cannot find siteName %s in the sitelist" % siteName) else: #Add siteName to DESIRED_Sites if ExtDESIRED_Sites has it (moving back to Normal) if siteName not in desiredSites and siteName in extDesiredSites: usi = desiredSites usi.append(siteName) usi = ','.join(map(str, usi)) sd.edit('WMAgent_JobID == %i' % jobID, "DESIRED_Sites", classad.ExprTree('"%s"' % usi)) else: #If job doesn't have the siteName in the siteList, just ignore it logging.debug( "Cannot find siteName %s in the sitelist" % siteName) return jobtokill
def addCRABInfoToClassAd(ad, info): """ Given a submit ClassAd, add in the appropriate CRAB_* attributes from the info directory """ for adName, dictName in SUBMIT_INFO: ad[adName] = classad.ExprTree(str(info[dictName]))
def test_ad_refs(self): ad = classad.ClassAd() ad["foo"] = classad.ExprTree("bar + baz") ad["bar"] = 2.1 ad["baz"] = 4 self.assertEqual(ad["foo"].__repr__(), "bar + baz") self.assertEqual(ad.eval("foo"), 6.1)
def test_operator(self): expr = classad.Literal(1) + 2 self.assertTrue(isinstance(expr, classad.ExprTree)) self.assertTrue(expr) self.assertTrue(expr.sameAs(classad.ExprTree('1 + 2'))) expr = classad.Literal(1) & 2 self.assertTrue(isinstance(expr, classad.ExprTree)) self.assertEquals(expr.eval(), 0) self.assertTrue(expr.sameAs(classad.ExprTree('1 & 2'))) expr = classad.Attribute("foo").is_(classad.Value.Undefined) self.assertTrue(expr.eval()) ad = classad.ClassAd("[foo = 1]") expr = classad.Attribute("foo").isnt_(classad.Value.Undefined) self.assertTrue(expr.eval(ad)) expr = classad.Literal(1).and_(classad.Literal(2)) self.assertRaises(RuntimeError, bool, expr)
def makeResizeAds(config): policies = {} for workflow, info in config.get('resizing', {}).items(): minCores = info.get("minCores", 3) maxCores = info.get("maxCores", 8) memoryPerThread = info.get("memoryPerThread") workflows = policies.setdefault((minCores, maxCores, memoryPerThread), set()) workflows.add(workflow) for policy, workflows in policies.items(): minCores, maxCores, memoryPerThread = policy anAd = classad.ClassAd() anAd['GridResource'] = 'condor localhost localhost' anAd['TargetUniverse'] = 5 # Same trick as above to convert the set to a ClassAd list. anAd["OverflowTasknames"] = map(str, workflows) tasks_escaped = anAd.lookup('OverflowTasknames').__repr__() del anAd['OverflowTaskNames'] anAd['Name'] = 'Resize Jobs (%d-%d cores, %d MB/thread)' % ( minCores, maxCores, memoryPerThread) anAd['Requirements'] = classad.ExprTree( '(target.WMCore_ResizeJob is False) && member(target.WMAgent_SubTaskName, %s)' % tasks_escaped) anAd['set_WMCore_ResizeJob'] = True anAd['set_MinCores'] = minCores anAd['set_MaxCores'] = maxCores anAd['set_HasBeenRouted'] = False anAd['set_ExtraMemory'] = memoryPerThread #anAd['set_RequestMemory'] = classad.ExprTree('OriginalMemory + %d * ( WMCore_ResizeJob ? ( RequestCpus - OriginalCpus ) : 0 )' % memoryPerThread) print anAd
def get_schedds(args=None, collectors=None): """ Return a list of schedd ads representing all the schedds in the pool. """ collectors = collectors or [ "cmssrv221.fnal.gov:9620", "cmsgwms-collector-tier0.cern.ch:9620", "cmssrv276.fnal.gov" ] schedd_query = classad.ExprTree('!isUndefined(CMSGWMS_Type)') schedd_ads = {} for host in collectors: coll = htcondor.Collector(host) try: schedds = coll.query( htcondor.AdTypes.Schedd, schedd_query, projection=["MyAddress", "ScheddIpAddr", "Name"]) except IOError, e: logging.warning(str(e)) continue for schedd in schedds: try: schedd_ads[schedd['Name']] = schedd except KeyError: pass
def _combine( self, other: Union["ConstraintHandle", classad.ExprTree, str], combinator ): if isinstance(other, ConstraintHandle) and ( self.collector != other.collector or self.scheduler != other.scheduler ): raise exceptions.InvalidHandle( "Cannot construct a handle for separate schedds" ) if isinstance(other, ConstraintHandle): c = other.constraint elif isinstance(other, classad.ExprTree): c = other elif isinstance(other, str): c = classad.ExprTree(other) else: raise exceptions.InvalidHandle( f"Cannot construct a combined handle from {self} and {other} because it is not a ConstraintHandle, ExprTree, or cannot be parsed into an ExprTree" ) return ConstraintHandle( combinator(self.constraint, c), collector=self.collector, scheduler=self.scheduler, )
def get_schedds(): #schedd_query = classad.ExprTree('CMSGWMS_Type =?="prodschedd" && Name =!= "vocms001.cern.ch" && Name =!= "vocms047.cern.ch" && Name=!="vocms015.cern.ch"') schedd_query = classad.ExprTree('!isUndefined(CMSGWMS_Type)') coll = htcondor.Collector("cmssrv221.fnal.gov:9620") schedd_ads = coll.query(htcondor.AdTypes.Schedd, schedd_query, projection=["MyAddress", "ScheddIpAddr", "Name"]) random.shuffle(schedd_ads) return schedd_ads
def test_ad_special_values(self): ad = classad.ClassAd() ad["foo"] = classad.ExprTree('regexp(12, 34)') ad["bar"] = classad.Value.Undefined self.assertEqual(ad["foo"].eval(), classad.Value.Error) self.assertNotEqual(ad["foo"].eval(), ad["bar"]) self.assertEqual(classad.Value.Undefined, ad["bar"])
def test_register(self): class BadException(Exception): pass def myAdd(a, b): return a + b def myBad(a, b): raise BadException("bad") def myComplex(a): return 1j # ClassAds have no complex numbers, not able to convert from python to an expression def myExpr(**kw): return classad.ExprTree( "foo" ) # Functions must return values; this becomes "undefined". def myFoo(foo): return foo['foo'] def myIntersect(a, b): return set(a).intersection(set(b)) classad.register(myAdd) classad.register(myAdd, name='myAdd2') classad.register(myBad) classad.register(myComplex) classad.register(myExpr) classad.register(myFoo) classad.register(myIntersect) self.assertEqual(3, classad.ExprTree('myAdd(1, 2)').eval()) self.assertEqual(3, classad.ExprTree('myAdd2(1, 2)').eval()) self.assertRaises(BadException, classad.ExprTree('myBad(1, 2)').eval) self.assertRaises(TypeError, classad.ExprTree('myComplex(1)').eval) self.assertEqual(classad.Value.Undefined, classad.ExprTree('myExpr()').eval()) self.assertEqual( classad.ExprTree('myExpr()').eval(classad.ClassAd({"foo": 2})), 2) self.assertRaises(TypeError, classad.ExprTree('myAdd(1)').eval ) # myAdd requires 2 arguments; only one is given. self.assertEqual(classad.ExprTree('myFoo([foo = 1])').eval(), 1) self.assertEqual( classad.ExprTree('size(myIntersect({1, 2}, {2, 3}))').eval(), 1) self.assertEqual( classad.ExprTree('myIntersect({1, 2}, {2, 3})[0]').eval(), 2)
def test_proc_expr(self): """get_num_procs() should be able to handle attributes set to ClassAd expressions """ for attr in condor_meter.PROC_ATTRS: jobad = classad.ClassAd() jobad[attr] = classad.ExprTree('2 + 2') procs = condor_meter.get_num_procs(jobad) self.assertEquals(procs, 4)
def test_state(self): def myFunc(state): return 1 if state else 0 classad.register(myFunc) self.assertEqual(0, classad.ExprTree('myFunc(false)').eval()) self.assertEqual(1, classad.ExprTree('myFunc("foo")').eval()) ad = classad.ClassAd("""[foo = myFunc(); bar = 2]""") self.assertEqual(1, ad.eval('foo')) ad['foo'] = classad.ExprTree('myFunc(1)') self.assertRaises(TypeError, ad.eval, ('foo', )) def myFunc(arg1, **kw): return kw['state']['bar'] classad.register(myFunc) self.assertEqual(2, ad.eval('foo'))
def execute(self): """Execute / submit a job with HTCondor.""" os.chdir(self.workflow_workspace) job_ad = classad.ClassAd() job_ad["JobDescription"] = ( self.workflow.get_full_workflow_name() + "_" + self.job_name ) job_ad["JobMaxRetries"] = 3 job_ad["LeaveJobInQueue"] = classad.ExprTree( "(JobStatus == 4) && ((StageOutFinish =?= UNDEFINED) || " "(StageOutFinish == 0))" ) job_ad["Cmd"] = ( "./job_wrapper.sh" if not self.unpacked_img else "./job_singularity_wrapper.sh" ) if not self.unpacked_img: job_ad["Arguments"] = self._format_arguments() job_ad["DockerImage"] = self.docker_img job_ad["WantDocker"] = True job_ad["Environment"] = self._format_env_vars() job_ad["Out"] = classad.ExprTree( 'strcat("reana_job.", ClusterId, ".", ProcId, ".out")' ) job_ad["Err"] = classad.ExprTree( 'strcat("reana_job.", ClusterId, ".", ProcId, ".err")' ) job_ad["log"] = classad.ExprTree('strcat("reana_job.", ClusterId, ".err")') job_ad["ShouldTransferFiles"] = "YES" job_ad["WhenToTransferOutput"] = "ON_EXIT" job_ad["TransferInput"] = self._get_input_files() job_ad["TransferOutput"] = "." job_ad["PeriodicRelease"] = classad.ExprTree("(HoldReasonCode == 35)") if self.htcondor_max_runtime in HTCONDOR_JOB_FLAVOURS.keys(): job_ad["JobFlavour"] = self.htcondor_max_runtime elif str.isdigit(self.htcondor_max_runtime): job_ad["MaxRunTime"] = int(self.htcondor_max_runtime) else: job_ad["MaxRunTime"] = 3600 if self.htcondor_accounting_group: job_ad["AccountingGroup"] = self.htcondor_accounting_group future = current_app.htcondor_executor.submit(self._submit, job_ad) clusterid = future.result() return clusterid
def test_register_custom_function(): def concatenateLists(list1, list2): return list1 + list2 classad.register(concatenateLists) assert classad.ExprTree("concatenateLists({1, 2}, {3, 4})").eval() == [ 1, 2, 3, 4 ]
def test_custom_function_can_see_python_values(): local = 1 def add(a): return a + local classad.register(add) assert classad.ExprTree("add(1)").eval() == 2
def evalExpressionStr(expression_str, context): """Evaluate a classad expression (in a string) in the given context (a ClassAd). Can raise: - SyntaxError (if expression_str is unparseable) """ return context.flatten(classad.ExprTree(expression_str))
def test_pickle(self): ad = classad.ClassAd({"one": 1}) expr = classad.ExprTree("2+2") pad = pickle.dumps(ad) pexpr = pickle.dumps(expr) ad2 = pickle.loads(pad) expr2 = pickle.loads(pexpr) self.assertEquals(ad2.__repr__(), "[ one = 1 ]") self.assertEquals(expr2.__repr__(), "2 + 2")
def getClusterAd(): """ Mimic the same method of SimpleCondorPlugin """ ad = classad.ClassAd() # ad['universe'] = "vanilla" ad['ShouldTransferFiles'] = "YES" ad['WhenToTransferOutput'] = "ON_EXIT" ad['UserLogUseXML'] = True ad['JobNotification'] = 0 ad['Cmd'] = TEST_DIR + 'submit_fake.sh' # Investigate whether we should pass the absolute path for Out and Err ads, # just as we did for UserLog. There may be issues, more info on WMCore #7362 ad['Out'] = classad.ExprTree( 'strcat("condor.", ClusterId, ".", ProcId, ".out")') ad['Err'] = classad.ExprTree( 'strcat("condor.", ClusterId, ".", ProcId, ".err")') ad['UserLog'] = classad.ExprTree( 'strcat(Iwd, "/condor.", ClusterId, ".", ProcId, ".log")') ad['WMAgent_AgentName'] = 'WMAgentCommissioning' ad['JobLeaseDuration'] = classad.ExprTree( 'isUndefined(MachineAttrMaxHibernateTime0) ? 1200 : MachineAttrMaxHibernateTime0' ) ad['PeriodicRemove'] = classad.ExprTree( '( JobStatus =?= 5 ) && ( time() - EnteredCurrentStatus > 10 * 60 )') removeReasonExpr = 'PeriodicRemove ? "Job automatically removed for being in Held status" : ""' ad['PeriodicRemoveReason'] = classad.ExprTree(removeReasonExpr) # Required for global pool accounting ad['AcctGroup'] = 'production' ad['AcctGroupUser'] = '******' ad['AccountingGroup'] = "%s.%s" % (ad['AcctGroup'], ad['AcctGroupUser']) # Customized classAds for this plugin ad['DESIRED_Archs'] = "INTEL,X86_64" ad['Rank'] = 0.0 ad['TransferIn'] = False ad['JobMachineAttrs'] = "GLIDEIN_CMSSite" ad['JobAdInformationAttrs'] = ( "JobStatus,QDate,EnteredCurrentStatus,JobStartDate,DESIRED_Sites," "ExtDESIRED_Sites,WMAgent_JobID,MachineAttrGLIDEIN_CMSSite0") # TODO: remove when 8.5.7 is deployed paramsToAdd = htcondor.param['SUBMIT_ATTRS'].split( ) + htcondor.param['SUBMIT_EXPRS'].split() paramsToSkip = [ 'accounting_group', 'use_x509userproxy', 'PostJobPrio2', 'JobAdInformationAttrs' ] for param in paramsToAdd: if (param not in ad) and (param in htcondor.param) and (param not in paramsToSkip): ad[param] = classad.ExprTree(htcondor.param[param]) return ad
def makePerformanceCorrectionsAds(configs): m_config = configs.get('memory', {}) for memory in m_config: wfs = m_config[memory] anAd = classad.ClassAd() anAd["GridResource"] = "condor localhost localhost" anAd["TargetUniverse"] = 5 anAd["Name"] = str("Set memory requirement to %s" % memory) anAd["MemoryTasknames"] = map(str, wfs) memory_names_escaped = anAd.lookup('MemoryTasknames').__repr__() exp = classad.ExprTree( 'member(target.WMAgent_SubTaskName, %s) && ((target.HasBeenMemoryTuned =!= true) || (target.OriginalMemory =!= %d))' % (memory_names_escaped, int(memory))) ## just set to a different value anAd["Requirements"] = classad.ExprTree(str(exp)) anAd['set_HasBeenMemoryTuned'] = True anAd['set_HasBeenRouted'] = False anAd['set_OriginalMemory'] = int(memory) print anAd t_config = configs.get('time', {}) for timing in t_config: wfs = t_config[timing] anAd = classad.ClassAd() anAd["GridResource"] = "condor localhost localhost" anAd["TargetUniverse"] = 5 anAd["Name"] = str("Set timing requirement to %s" % timing) anAd["TimeTasknames"] = map(str, wfs) time_names_escaped = anAd.lookup('TimeTasknames').__repr__() exp = classad.ExprTree( 'member(target.WMAgent_SubTaskName, %s) && ((target.HasBeenTimingTuned =!= true) || (target.EstimatedSingleCoreMins <= %d))' % (time_names_escaped, int(timing))) anAd["Requirements"] = classad.ExprTree(str(exp)) anAd['set_HasBeenTimingTuned'] = True anAd['set_HasBeenRouted'] = False anAd['set_EstimatedSingleCoreMins'] = int(timing) anAd['set_OriginalMaxWallTimeMins'] = classad.ExprTree( 'EstimatedSingleCoreMins / OriginalCpus') print anAd s_config = configs.get('slope', {}) for slope in s_config: wfs = s_config[slope] anAd = classad.ClassAd() anAd["GridResource"] = "condor localhost localhost" anAd["TargetUniverse"] = 5 anAd["Name"] = str("Set memory per thread requirement to %s" % slope) anAd["TimeTasknames"] = map(str, wfs) time_names_escaped = anAd.lookup('TimeTasknames').__repr__() exp = classad.ExprTree( 'member(target.WMAgent_SubTaskName, %s) && (target.ExtraMemory =!= %d)' % (time_names_escaped, int(slope))) ## just set to a different value anAd["Requirements"] = classad.ExprTree(str(exp)) anAd['set_HasBeenSlopeTuned'] = True anAd['set_HasBeenRouted'] = False anAd['set_ExtraMemory'] = int(slope) print anAd
def test_register_with_exception(): def bad(a, b): raise Exception("oops") classad.register(bad) with pytest.raises(Exception) as e: classad.ExprTree("bad(1, 2)").eval() assert str(e.value) == "oops"
def makeReadAds(config): for needs, tasks in config.get('read', {}).items(): anAd = classad.ClassAd() set_read = int(float(needs)) anAd["Name"] = str("Set read requirement to %s" % set_read) anAd["GridResource"] = "condor localhost localhost" anAd["TargetUniverse"] = 5 anAd["JobRouterTasknames"] = map(str, tasks) task_names_escaped = anAd.lookup('JobRouterTasknames').__repr__() del anAd["JobRouterTasknames"] exp = classad.ExprTree( 'member(target.WMAgent_SubTaskName, %s) && (EstimatedInputRateKBs =!= %d)' % (task_names_escaped, int(set_read))) ## just set to a different value anAd["Requirements"] = classad.ExprTree(str(exp)) anAd["set_HasBeenRouted"] = False anAd["set_HasBeenReadTuned"] = True anAd["set_EstimatedInputRateKBs"] = int(set_read) print anAd
def __when_preemptible__(): try: return int( classad.ExprTree( htcondor.param.get('IDLE_WARNING_TRIGGER')).eval()) except Exception as e: log.critical("""Could not find or evaluate IDLE_WARNING_TRIGGER in HTCondor configuration. Returning None. Exception: {0}""".format(e)) return None
def test_list_conversion(self): ad = dict(classad.ClassAd("[a = {1,2,3}]")) self.assertTrue(isinstance(ad["a"], types.ListType)) self.assertTrue(isinstance(ad["a"][0], types.LongType)) def listAdd(a, b): return a + b classad.register(listAdd) self.assertEqual(classad.ExprTree("listAdd({1,2}, {3,4})")[0], 1)