def sitesReq(self, sites):
fmt = lambda x: 'RegExp(%s, other.GlueCEUniqueID)' % jdlEscape(x)
(blacklist, whitelist) = utils.splitBlackWhiteList(sites)
sitereqs = map(lambda x: '!' + fmt(x), blacklist)
if len(whitelist):
sitereqs.append('(%s)' % str.join(' || ', map(fmt, whitelist)))
if sitereqs:
return '( %s )' % str.join(' && ', sitereqs)
def _format_reqs_sites(self, sites):
fmt = lambda x: 'RegExp(%s, other.GlueCEUniqueID)' % self._escape(x)
(blacklist, whitelist) = utils.splitBlackWhiteList(sites)
sitereqs = lmap(lambda x: '!' + fmt(x), blacklist)
if whitelist:
sitereqs.append('(%s)' % str.join(' || ', imap(fmt, whitelist)))
if sitereqs:
return '( %s )' % str.join(' && ', sitereqs)
def _format_reqs_sites(self, sites):
fmt = lambda x: 'RegExp(%s, other.GlueCEUniqueID)' % self._escape(x)
(blacklist, whitelist) = utils.splitBlackWhiteList(sites)
sitereqs = lmap(lambda x: '!' + fmt(x), blacklist)
if whitelist:
sitereqs.append('(%s)' % str.join(' || ', imap(fmt, whitelist)))
if sitereqs:
return '( %s )' % str.join(' && ', sitereqs)
def _getFormattedRequirements(self, jobNum, task):
jdlReq=[]
# get requirements from task and broker WMS sites
reqs = self.brokerSite.brokerAdd(task.getRequirements(jobNum), WMS.SITES)
for reqType, reqValue in reqs:
if reqType == WMS.SITES:
(refuseSites, desireSites) = utils.splitBlackWhiteList(reqValue[1])
#(blacklist, whitelist) = utils.splitBlackWhiteList(reqValue[1])
## sites matching regular expression requirements
#refuseRegx=[ site for site in self._siteMap.keys()
# if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in blacklist ] ]
#desireRegx=[ site for site in self._siteMap.keys()
# if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in whitelist ] ]
## sites specifically matched
#refuseSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), blacklist) ]
#desireSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), whitelist) ]
## sites to actually match; refusing takes precedence over desiring, specific takes precedence over expression
#refuseSites=set(refuseSite).union(set(refuseRegx))
#desireSites=set(desireSite).union(set(desireRegx)-set(refuseRegx))-set(refuseSite)
if "blacklistSite" in self.poolReqs:
jdlReq.append( self.poolReqs["blacklistSite"] + ' = ' + '"' + ','.join(refuseSites) + '"' )
if "whitelistSite" in self.poolReqs:
jdlReq.append( self.poolReqs["whitelistSite"] + ' = ' + '"' + ','.join(desireSites) + '"' )
elif reqType == WMS.WALLTIME:
if ("walltimeMin" in self.poolReqs) and reqValue > 0:
jdlReq.append('%s = %d' % (self.poolReqs["walltimeMin"], reqValue))
elif reqType == WMS.STORAGE:
if ("requestSEs" in self.poolReqs):
jdlReq.append( self.poolReqs["requestSEs"] + ' = ' + '"' + ','.join(reqValue) + '"' )
elif reqType == WMS.MEMORY and reqValue > 0:
jdlReq.append('request_memory = %dM' % reqValue)
elif reqType == WMS.CPUS and reqValue > 0:
jdlReq.append('request_cpus = %d' % reqValue)
#append unused requirements to JDL for debugging
elif self.debug:
self.debugOut("reqType: %s reqValue: %s" % (reqType,reqValue))
self.debugFlush()
jdlReq.append('# Unused Requirement:')
jdlReq.append('# Type: %s' % reqType )
jdlReq.append('# Value: %s' % reqValue )
#TODO::: GLIDEIN_REQUIRE_GLEXEC_USE, WMS.SOFTWARE
# (HPDA) file location service
if "dataFiles" in self.poolReqs:
jdlReq.extend(self._getRequirementsFileList(jobNum, task))
return jdlReq
def _getFormattedRequirements(self, jobNum, task):
jdlReq=[]
# get requirements from task and broker WMS sites
reqs = self.brokerSite.brokerAdd(task.getRequirements(jobNum), WMS.SITES)
for reqType, reqValue in reqs:
if reqType == WMS.SITES:
(refuseSites, desireSites) = utils.splitBlackWhiteList(reqValue[1])
#(blacklist, whitelist) = utils.splitBlackWhiteList(reqValue[1])
## sites matching regular expression requirements
#refuseRegx=[ site for site in self._siteMap.keys()
# if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in blacklist ] ]
#desireRegx=[ site for site in self._siteMap.keys()
# if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in whitelist ] ]
## sites specifically matched
#refuseSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), blacklist) ]
#desireSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), whitelist) ]
## sites to actually match; refusing takes precedence over desiring, specific takes precedence over expression
#refuseSites=set(refuseSite).union(set(refuseRegx))
#desireSites=set(desireSite).union(set(desireRegx)-set(refuseRegx))-set(refuseSite)
if "blacklistSite" in self.poolReqs:
jdlReq.append( self.poolReqs["blacklistSite"] + ' = ' + '"' + ','.join(refuseSites) + '"' )
if "whitelistSite" in self.poolReqs:
jdlReq.append( self.poolReqs["whitelistSite"] + ' = ' + '"' + ','.join(desireSites) + '"' )
elif reqType == WMS.WALLTIME:
if ("walltimeMin" in self.poolReqs) and reqValue > 0:
jdlReq.append('%s = %d' % (self.poolReqs["walltimeMin"], reqValue))
elif reqType == WMS.STORAGE:
if ("requestSEs" in self.poolReqs):
jdlReq.append( self.poolReqs["requestSEs"] + ' = ' + '"' + ','.join(reqValue) + '"' )
elif reqType == WMS.MEMORY and reqValue > 0:
jdlReq.append('request_memory = %dM' % reqValue)
elif reqType == WMS.CPUS and reqValue > 0:
jdlReq.append('request_cpus = %d' % reqValue)
#append unused requirements to JDL for debugging
elif self.debug:
self.debugOut("reqType: %s reqValue: %s" % (reqType,reqValue))
self.debugFlush()
jdlReq.append('# Unused Requirement:')
jdlReq.append('# Type: %s' % reqType )
jdlReq.append('# Value: %s' % reqValue )
#TODO::: GLIDEIN_REQUIRE_GLEXEC_USE, WMS.SOFTWARE
# (HPDA) file location service
if "dataFiles" in self.poolReqs:
jdlReq.extend(self._getRequirementsFileList(jobNum, task))
return jdlReq
def _getRequirementJdlData(self, task, jobNum):
"""Create JDL attributes corresponding to job requirements"""
jdlData = []
requirements = task.getRequirements(jobNum)
poolRequMap = self.parentPool.jdlRequirementMap
for reqType, reqValue in requirements:
# ('WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE')
if reqType == WMS.SITES:
(wantSites, vetoSites) = utils.splitBlackWhiteList(reqValue[1])
if "+SITES" in poolRequMap:
jdlData.append( '%s = "%s"' % (
poolRequMap["+SITES"][0],
poolRequMap["+SITES"][1] % ','.join(wantSites)
)
)
if "-SITES" in poolRequMap:
jdlData.append( '%s = "%s"' % (
poolRequMap["-SITES"][0],
poolRequMap["-SITES"][1] % ','.join(vetoSites)
)
)
continue
if reqType == WMS.STORAGE:
if ("STORAGE" in poolRequMap) and reqValue > 0:
jdlData.append( '%s = %s ' % (
poolRequMap["STORAGE"][0],
poolRequMap["STORAGE"][1] % ','.join(reqValue)
)
)
continue
#HACK
if reqValue > 0 and WMS.reqTypes[reqType] in poolRequMap:
jdlData.append( "%s = %s" % (
poolRequMap[WMS.reqTypes[reqType]][0],
poolRequMap[WMS.reqTypes[reqType]][1] % reqValue
)
)
continue
try:
if int(reqValue) <= 0:
continue
except TypeError:
pass
self._log(logging.INFO3, "Requirement '%s' cannot be mapped to pool and will be ignored!" % WMS.reqTypes[reqType])
return jdlData
def _getRequirementJdlData(self, task, jobNum):
"""Create JDL attributes corresponding to job requirements"""
jdlData = []
requirements = task.getRequirements(jobNum)
poolRequMap = self.parentPool.jdlRequirementMap
for reqType, reqValue in requirements:
# ('WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE')
if reqType == WMS.SITES:
(wantSites, vetoSites) = utils.splitBlackWhiteList(reqValue[1])
if "+SITES" in poolRequMap:
jdlData.append(
'%s = "%s"' %
(poolRequMap["+SITES"][0],
poolRequMap["+SITES"][1] % ','.join(wantSites)))
if "-SITES" in poolRequMap:
jdlData.append(
'%s = "%s"' %
(poolRequMap["-SITES"][0],
poolRequMap["-SITES"][1] % ','.join(vetoSites)))
continue
if reqType == WMS.STORAGE:
if ("STORAGE" in poolRequMap) and reqValue > 0:
jdlData.append(
'%s = %s ' %
(poolRequMap["STORAGE"][0],
poolRequMap["STORAGE"][1] % ','.join(reqValue)))
continue
#HACK
if reqValue > 0 and WMS.reqTypes[reqType] in poolRequMap:
jdlData.append(
"%s = %s" %
(poolRequMap[WMS.reqTypes[reqType]][0],
poolRequMap[WMS.reqTypes[reqType]][1] % reqValue))
continue
try:
if int(reqValue) <= 0:
continue
except TypeError:
clear_current_exception()
self._log(
logging.INFO3,
"Requirement '%s' cannot be mapped to pool and will be ignored!"
% WMS.reqTypes[reqType])
return jdlData
def _getFormattedRequirements(self, jobNum, task):
jdlReq = []
# get requirements from task and broker WMS sites
reqs = self.brokerSite.brokerAdd(task.getRequirements(jobNum),
WMS.SITES)
for reqType, reqValue in reqs:
if reqType == WMS.SITES:
(refuseSites,
desireSites) = utils.splitBlackWhiteList(reqValue[1])
#(blacklist, whitelist) = utils.splitBlackWhiteList(reqValue[1])
## sites matching regular expression requirements
#refuseRegx=[ site for site in self._siteMap.keys()
# if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in blacklist ] ]
#desireRegx=[ site for site in self._siteMap.keys()
# if True in [ re.search(bexpr.lower(),siteDescript.lower()) is not None for siteDescript in _siteMap[site] for bexpr in whitelist ] ]
## sites specifically matched
#refuseSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), blacklist) ]
#desireSite=[ site for site in self._siteMap.keys() if site.lower() in ap(lambda req: req.lower(), whitelist) ]
## sites to actually match; refusing takes precedence over desiring, specific takes precedence over expression
#refuseSites=set(refuseSite).union(set(refuseRegx))
#desireSites=set(desireSite).union(set(desireRegx)-set(refuseRegx))-set(refuseSite)
if "blacklistSite" in self.poolReqs:
jdlReq.append(self.poolReqs["blacklistSite"] + ' = ' +
'"' + ','.join(refuseSites) + '"')
if "whitelistSite" in self.poolReqs:
jdlReq.append(self.poolReqs["whitelistSite"] + ' = ' +
'"' + ','.join(desireSites) + '"')
elif reqType == WMS.WALLTIME:
if ("walltimeMin" in self.poolReqs) and reqValue > 0:
jdlReq.append('%s = %d' %
(self.poolReqs["walltimeMin"], reqValue))
elif reqType == WMS.STORAGE:
if ("requestSEs" in self.poolReqs):
jdlReq.append(self.poolReqs["requestSEs"] + ' = ' + '"' +
','.join(reqValue) + '"')
#append unused requirements to JDL for debugging
elif self.debug:
self.debugOut("reqType: %s reqValue: %s" %
(reqType, reqValue))
self.debugFlush()
jdlReq.append('# Unused Requirement:')
jdlReq.append('# Type: %s' % reqType)
jdlReq.append('# Type: %s' % reqValue)
#TODO::: GLIDEIN_REQUIRE_GLEXEC_USE, WMS.SOFTWARE, WMS.MEMORY, WMS.CPUS
# (HPDA) file location service
if "dataFiles" in self.poolReqs:
# as per ``formatFileList``
# UserMod filelists are space separated 'File1 File2 File3'
# CMSSW filelists are individually quoted and comma+space separated '"File1", "File2", "File3"'
file_list = task.getJobConfig(jobNum).get('FILE_NAMES', '').strip()
if '", "' in file_list: # CMSSW style
file_list = file_list.strip('"').split('", "')
else: # UserMod style
file_list = file_list.split(' ')
if file_list:
arg_key = self.poolReqs["dataFiles"]
data_file = os.path.join(self.getSandboxPath(jobNum),
'job_%d_files.txt' % jobNum)
data_file_list = open(data_file, "w")
try:
data_file_list.writelines(
lmap(lambda line: line + "\n", file_list))
finally:
data_file_list.close()
jdlReq.append('%s = "%s"' % (arg_key, data_file))
return jdlReq
