def sitesReq(self, sites): fmt = lambda x: 'RegExp(%s, other.GlueCEUniqueID)' % jdlEscape(x) (blacklist, whitelist) = utils.splitBlackWhiteList(sites) sitereqs = map(lambda x: '!' + fmt(x), blacklist) if len(whitelist): sitereqs.append('(%s)' % str.join(' || ', map(fmt, whitelist))) if sitereqs: return '( %s )' % str.join(' && ', sitereqs)
def _format_reqs_sites(self, sites): fmt = lambda x: 'RegExp(%s, other.GlueCEUniqueID)' % self._escape(x) (blacklist, whitelist) = utils.splitBlackWhiteList(sites) sitereqs = lmap(lambda x: '!' + fmt(x), blacklist) if whitelist: sitereqs.append('(%s)' % str.join(' || ', imap(fmt, whitelist))) if sitereqs: return '( %s )' % str.join(' && ', sitereqs)
def _getFormattedRequirements(self, jobNum, task): jdlReq=[] # get requirements from task and broker WMS sites reqs = self.brokerSite.brokerAdd(task.getRequirements(jobNum), WMS.SITES) for reqType, reqValue in reqs: if reqType == WMS.SITES: (refuseSites, desireSites) = utils.splitBlackWhiteList(reqValue[1]) #(blacklist, whitelist) = utils.splitBlackWhiteList(reqValue[1]) ## sites matching regular expression requirements #refuseRegx=[ site for site in self._siteMap.keys() # if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in blacklist ] ] #desireRegx=[ site for site in self._siteMap.keys() # if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in whitelist ] ] ## sites specifically matched #refuseSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), blacklist) ] #desireSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), whitelist) ] ## sites to actually match; refusing takes precedence over desiring, specific takes precedence over expression #refuseSites=set(refuseSite).union(set(refuseRegx)) #desireSites=set(desireSite).union(set(desireRegx)-set(refuseRegx))-set(refuseSite) if "blacklistSite" in self.poolReqs: jdlReq.append( self.poolReqs["blacklistSite"] + ' = ' + '"' + ','.join(refuseSites) + '"' ) if "whitelistSite" in self.poolReqs: jdlReq.append( self.poolReqs["whitelistSite"] + ' = ' + '"' + ','.join(desireSites) + '"' ) elif reqType == WMS.WALLTIME: if ("walltimeMin" in self.poolReqs) and reqValue > 0: jdlReq.append('%s = %d' % (self.poolReqs["walltimeMin"], reqValue)) elif reqType == WMS.STORAGE: if ("requestSEs" in self.poolReqs): jdlReq.append( self.poolReqs["requestSEs"] + ' = ' + '"' + ','.join(reqValue) + '"' ) elif reqType == WMS.MEMORY and reqValue > 0: jdlReq.append('request_memory = %dM' % reqValue) elif reqType == WMS.CPUS and reqValue > 0: jdlReq.append('request_cpus = %d' % reqValue) #append unused requirements to JDL for debugging elif self.debug: self.debugOut("reqType: %s reqValue: %s" % (reqType,reqValue)) self.debugFlush() jdlReq.append('# Unused Requirement:') jdlReq.append('# Type: %s' % reqType ) jdlReq.append('# Value: %s' % reqValue ) #TODO::: GLIDEIN_REQUIRE_GLEXEC_USE, WMS.SOFTWARE # (HPDA) file location service if "dataFiles" in self.poolReqs: jdlReq.extend(self._getRequirementsFileList(jobNum, task)) return jdlReq
def _getRequirementJdlData(self, task, jobNum): """Create JDL attributes corresponding to job requirements""" jdlData = [] requirements = task.getRequirements(jobNum) poolRequMap = self.parentPool.jdlRequirementMap for reqType, reqValue in requirements: # ('WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE') if reqType == WMS.SITES: (wantSites, vetoSites) = utils.splitBlackWhiteList(reqValue[1]) if "+SITES" in poolRequMap: jdlData.append( '%s = "%s"' % ( poolRequMap["+SITES"][0], poolRequMap["+SITES"][1] % ','.join(wantSites) ) ) if "-SITES" in poolRequMap: jdlData.append( '%s = "%s"' % ( poolRequMap["-SITES"][0], poolRequMap["-SITES"][1] % ','.join(vetoSites) ) ) continue if reqType == WMS.STORAGE: if ("STORAGE" in poolRequMap) and reqValue > 0: jdlData.append( '%s = %s ' % ( poolRequMap["STORAGE"][0], poolRequMap["STORAGE"][1] % ','.join(reqValue) ) ) continue #HACK if reqValue > 0 and WMS.reqTypes[reqType] in poolRequMap: jdlData.append( "%s = %s" % ( poolRequMap[WMS.reqTypes[reqType]][0], poolRequMap[WMS.reqTypes[reqType]][1] % reqValue ) ) continue try: if int(reqValue) <= 0: continue except TypeError: pass self._log(logging.INFO3, "Requirement '%s' cannot be mapped to pool and will be ignored!" % WMS.reqTypes[reqType]) return jdlData
def _getRequirementJdlData(self, task, jobNum): """Create JDL attributes corresponding to job requirements""" jdlData = [] requirements = task.getRequirements(jobNum) poolRequMap = self.parentPool.jdlRequirementMap for reqType, reqValue in requirements: # ('WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE') if reqType == WMS.SITES: (wantSites, vetoSites) = utils.splitBlackWhiteList(reqValue[1]) if "+SITES" in poolRequMap: jdlData.append( '%s = "%s"' % (poolRequMap["+SITES"][0], poolRequMap["+SITES"][1] % ','.join(wantSites))) if "-SITES" in poolRequMap: jdlData.append( '%s = "%s"' % (poolRequMap["-SITES"][0], poolRequMap["-SITES"][1] % ','.join(vetoSites))) continue if reqType == WMS.STORAGE: if ("STORAGE" in poolRequMap) and reqValue > 0: jdlData.append( '%s = %s ' % (poolRequMap["STORAGE"][0], poolRequMap["STORAGE"][1] % ','.join(reqValue))) continue #HACK if reqValue > 0 and WMS.reqTypes[reqType] in poolRequMap: jdlData.append( "%s = %s" % (poolRequMap[WMS.reqTypes[reqType]][0], poolRequMap[WMS.reqTypes[reqType]][1] % reqValue)) continue try: if int(reqValue) <= 0: continue except TypeError: clear_current_exception() self._log( logging.INFO3, "Requirement '%s' cannot be mapped to pool and will be ignored!" % WMS.reqTypes[reqType]) return jdlData
def _getFormattedRequirements(self, jobNum, task): jdlReq = [] # get requirements from task and broker WMS sites reqs = self.brokerSite.brokerAdd(task.getRequirements(jobNum), WMS.SITES) for reqType, reqValue in reqs: if reqType == WMS.SITES: (refuseSites, desireSites) = utils.splitBlackWhiteList(reqValue[1]) #(blacklist, whitelist) = utils.splitBlackWhiteList(reqValue[1]) ## sites matching regular expression requirements #refuseRegx=[ site for site in self._siteMap.keys() # if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in blacklist ] ] #desireRegx=[ site for site in self._siteMap.keys() # if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in whitelist ] ] ## sites specifically matched #refuseSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), blacklist) ] #desireSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), whitelist) ] ## sites to actually match; refusing takes precedence over desiring, specific takes precedence over expression #refuseSites=set(refuseSite).union(set(refuseRegx)) #desireSites=set(desireSite).union(set(desireRegx)-set(refuseRegx))-set(refuseSite) if "blacklistSite" in self.poolReqs: jdlReq.append(self.poolReqs["blacklistSite"] + ' = ' + '"' + ','.join(refuseSites) + '"') if "whitelistSite" in self.poolReqs: jdlReq.append(self.poolReqs["whitelistSite"] + ' = ' + '"' + ','.join(desireSites) + '"') elif reqType == WMS.WALLTIME: if ("walltimeMin" in self.poolReqs) and reqValue > 0: jdlReq.append('%s = %d' % (self.poolReqs["walltimeMin"], reqValue)) elif reqType == WMS.STORAGE: if ("requestSEs" in self.poolReqs): jdlReq.append(self.poolReqs["requestSEs"] + ' = ' + '"' + ','.join(reqValue) + '"') #append unused requirements to JDL for debugging elif self.debug: self.debugOut("reqType: %s reqValue: %s" % (reqType, reqValue)) self.debugFlush() jdlReq.append('# Unused Requirement:') jdlReq.append('# Type: %s' % reqType) jdlReq.append('# Type: %s' % reqValue) #TODO::: GLIDEIN_REQUIRE_GLEXEC_USE, WMS.SOFTWARE, WMS.MEMORY, WMS.CPUS # (HPDA) file location service if "dataFiles" in self.poolReqs: # as per ``formatFileList`` # UserMod filelists are space separated 'File1 File2 File3' # CMSSW filelists are individually quoted and comma+space separated '"File1", "File2", "File3"' file_list = task.getJobConfig(jobNum).get('FILE_NAMES', '').strip() if '", "' in file_list: # CMSSW style file_list = file_list.strip('"').split('", "') else: # UserMod style file_list = file_list.split(' ') if file_list: arg_key = self.poolReqs["dataFiles"] data_file = os.path.join(self.getSandboxPath(jobNum), 'job_%d_files.txt' % jobNum) data_file_list = open(data_file, "w") try: data_file_list.writelines( lmap(lambda line: line + "\n", file_list)) finally: data_file_list.close() jdlReq.append('%s = "%s"' % (arg_key, data_file)) return jdlReq