def _get_jdl_req_str_list(self, jobnum, task):
# helper for converting GC requirements to Condor requirements
jdl_req_str_list = []
def _add_list_classad(classad_name, value):
if classad_name in self._pool_req_dict:
classad_str = self._pool_req_dict[classad_name]
jdl_req_str_list.append('%s = "%s"' % (classad_str, str.join(', ', value)))
# get requirements from task and broker WMS sites
req_list = self._broker_site.broker(task.get_requirement_list(jobnum), WMS.SITES)
for req_type, req_value in req_list:
if req_type == WMS.SITES:
(blacklist, whitelist) = split_blackwhite_list(req_value[1])
_add_list_classad('blacklistSite', blacklist)
_add_list_classad('whitelistSite', whitelist)
elif req_type == WMS.WALLTIME:
if ('walltimeMin' in self._pool_req_dict) and (req_value > 0):
jdl_req_str_list.append('%s = %d' % (self._pool_req_dict['walltimeMin'], req_value))
elif (req_type == WMS.STORAGE) and req_value:
_add_list_classad('requestSEs', req_value)
elif (req_type == WMS.MEMORY) and (req_value > 0):
jdl_req_str_list.append('request_memory = %dM' % req_value)
elif (req_type == WMS.CPUS) and (req_value > 0):
jdl_req_str_list.append('request_cpus = %d' % req_value)
elif (req_type == WMS.DISKSPACE) and (req_value > 0):
jdl_req_str_list.append('request_disk = %d' % (1024 * req_value))
# TODO: GLIDEIN_REQUIRE_GLEXEC_USE, WMS.SOFTWARE
# (HPDA) file location service
if 'dataFiles' in self._pool_req_dict:
jdl_req_str_list.extend(self._get_dataset_fn_list(jobnum, task))
return jdl_req_str_list
def _get_jdl_req_str_list(self, jobnum, task):
# helper for converting GC requirements to Condor requirements
jdl_req_str_list = []
def _add_list_classad(classad_name, value):
if classad_name in self._pool_req_dict:
classad_str = self._pool_req_dict[classad_name]
jdl_req_str_list.append('%s = "%s"' % (classad_str, str.join(', ', value)))
# get requirements from task and broker WMS sites
req_list = self._broker_site.broker(task.get_requirement_list(jobnum), WMS.SITES)
for req_type, req_value in req_list:
if req_type == WMS.SITES:
(blacklist, whitelist) = split_blackwhite_list(req_value[1])
_add_list_classad('blacklistSite', blacklist)
_add_list_classad('whitelistSite', whitelist)
elif req_type == WMS.WALLTIME:
if ('walltime' in self._pool_req_dict) and (req_value > 0):
jdl_req_str_list.append('%s = %d' % (self._pool_req_dict['walltime'], req_value))
elif (req_type == WMS.STORAGE) and req_value:
_add_list_classad('requestSEs', req_value)
elif (req_type == WMS.MEMORY) and (req_value > 0):
jdl_req_str_list.append('request_memory = %dM' % req_value)
elif (req_type == WMS.CPUS) and (req_value > 0):
jdl_req_str_list.append('request_cpus = %d' % req_value)
elif (req_type == WMS.DISKSPACE) and (req_value > 0):
jdl_req_str_list.append('request_disk = %d' % (1024 * req_value))
# TODO: GLIDEIN_REQUIRE_GLEXEC_USE, WMS.SOFTWARE
# (HPDA) file location service
if 'dataFiles' in self._pool_req_dict:
jdl_req_str_list.extend(self._get_dataset_fn_list(jobnum, task))
return jdl_req_str_list
def _format_reqs_sites(self, sites):
def _fmt_sites(site):
return 'RegExp(%s, other.GlueCEUniqueID)' % self._escape(site)
(blacklist, whitelist) = split_blackwhite_list(sites)
sitereqs = lmap(lambda x: '!' + _fmt_sites(x), blacklist)
if whitelist:
sitereqs.append('(%s)' % str.join(' || ', imap(_fmt_sites, whitelist)))
if sitereqs:
return '( %s )' % str.join(' && ', sitereqs)
def _getRequirementJdlData(self, task, jobnum):
"""Create JDL attributes corresponding to job requirements"""
jdlData = []
requirements = task.get_requirement_list(jobnum)
poolRequMap = self.parentPool.jdlRequirementMap
for reqType, reqValue in requirements:
# ('WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE')
if reqType == WMS.SITES:
(wantSites, vetoSites) = split_blackwhite_list(reqValue[1])
if "+SITES" in poolRequMap:
jdlData.append( '%s = "%s"' % (
poolRequMap["+SITES"][0],
poolRequMap["+SITES"][1] % ','.join(wantSites)
)
)
if "-SITES" in poolRequMap:
jdlData.append( '%s = "%s"' % (
poolRequMap["-SITES"][0],
poolRequMap["-SITES"][1] % ','.join(vetoSites)
)
)
continue
if reqType == WMS.STORAGE:
if ("STORAGE" in poolRequMap) and reqValue > 0:
jdlData.append( '%s = %s ' % (
poolRequMap["STORAGE"][0],
poolRequMap["STORAGE"][1] % ','.join(reqValue)
)
)
continue
#HACK
if reqValue > 0 and WMS.reqTypes[reqType] in poolRequMap:
jdlData.append( "%s = %s" % (
poolRequMap[WMS.reqTypes[reqType]][0],
poolRequMap[WMS.reqTypes[reqType]][1] % reqValue
)
)
continue
try:
if int(reqValue) <= 0:
continue
except TypeError:
clear_current_exception()
self._log(logging.INFO3, "Requirement '%s' cannot be mapped to pool and will be ignored!" % WMS.reqTypes[reqType])
return jdlData
def _getRequirementJdlData(self, task, jobnum):
"""Create JDL attributes corresponding to job requirements"""
jdlData = []
requirements = task.get_requirement_list(jobnum)
poolRequMap = self.parentPool.jdlRequirementMap
for reqType, reqValue in requirements:
# ('WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE')
if reqType == WMS.SITES:
(wantSites, vetoSites) = split_blackwhite_list(reqValue[1])
if "+SITES" in poolRequMap:
jdlData.append(
'%s = "%s"' %
(poolRequMap["+SITES"][0],
poolRequMap["+SITES"][1] % ','.join(wantSites)))
if "-SITES" in poolRequMap:
jdlData.append(
'%s = "%s"' %
(poolRequMap["-SITES"][0],
poolRequMap["-SITES"][1] % ','.join(vetoSites)))
continue
if reqType == WMS.STORAGE:
if ("STORAGE" in poolRequMap) and reqValue > 0:
jdlData.append(
'%s = %s ' %
(poolRequMap["STORAGE"][0],
poolRequMap["STORAGE"][1] % ','.join(reqValue)))
continue
#HACK
if reqValue > 0 and WMS.reqTypes[reqType] in poolRequMap:
jdlData.append(
"%s = %s" %
(poolRequMap[WMS.reqTypes[reqType]][0],
poolRequMap[WMS.reqTypes[reqType]][1] % reqValue))
continue
try:
if int(reqValue) <= 0:
continue
except TypeError:
clear_current_exception()
self._log(
logging.INFO3,
"Requirement '%s' cannot be mapped to pool and will be ignored!"
% WMS.reqTypes[reqType])
return jdlData