def _get_jdl_req_str_list(self, jobnum, task): # helper for converting GC requirements to Condor requirements jdl_req_str_list = [] def _add_list_classad(classad_name, value): if classad_name in self._pool_req_dict: classad_str = self._pool_req_dict[classad_name] jdl_req_str_list.append('%s = "%s"' % (classad_str, str.join(', ', value))) # get requirements from task and broker WMS sites req_list = self._broker_site.broker(task.get_requirement_list(jobnum), WMS.SITES) for req_type, req_value in req_list: if req_type == WMS.SITES: (blacklist, whitelist) = split_blackwhite_list(req_value[1]) _add_list_classad('blacklistSite', blacklist) _add_list_classad('whitelistSite', whitelist) elif req_type == WMS.WALLTIME: if ('walltimeMin' in self._pool_req_dict) and (req_value > 0): jdl_req_str_list.append('%s = %d' % (self._pool_req_dict['walltimeMin'], req_value)) elif (req_type == WMS.STORAGE) and req_value: _add_list_classad('requestSEs', req_value) elif (req_type == WMS.MEMORY) and (req_value > 0): jdl_req_str_list.append('request_memory = %dM' % req_value) elif (req_type == WMS.CPUS) and (req_value > 0): jdl_req_str_list.append('request_cpus = %d' % req_value) elif (req_type == WMS.DISKSPACE) and (req_value > 0): jdl_req_str_list.append('request_disk = %d' % (1024 * req_value)) # TODO: GLIDEIN_REQUIRE_GLEXEC_USE, WMS.SOFTWARE # (HPDA) file location service if 'dataFiles' in self._pool_req_dict: jdl_req_str_list.extend(self._get_dataset_fn_list(jobnum, task)) return jdl_req_str_list
def _get_jdl_req_str_list(self, jobnum, task): # helper for converting GC requirements to Condor requirements jdl_req_str_list = [] def _add_list_classad(classad_name, value): if classad_name in self._pool_req_dict: classad_str = self._pool_req_dict[classad_name] jdl_req_str_list.append('%s = "%s"' % (classad_str, str.join(', ', value))) # get requirements from task and broker WMS sites req_list = self._broker_site.broker(task.get_requirement_list(jobnum), WMS.SITES) for req_type, req_value in req_list: if req_type == WMS.SITES: (blacklist, whitelist) = split_blackwhite_list(req_value[1]) _add_list_classad('blacklistSite', blacklist) _add_list_classad('whitelistSite', whitelist) elif req_type == WMS.WALLTIME: if ('walltime' in self._pool_req_dict) and (req_value > 0): jdl_req_str_list.append('%s = %d' % (self._pool_req_dict['walltime'], req_value)) elif (req_type == WMS.STORAGE) and req_value: _add_list_classad('requestSEs', req_value) elif (req_type == WMS.MEMORY) and (req_value > 0): jdl_req_str_list.append('request_memory = %dM' % req_value) elif (req_type == WMS.CPUS) and (req_value > 0): jdl_req_str_list.append('request_cpus = %d' % req_value) elif (req_type == WMS.DISKSPACE) and (req_value > 0): jdl_req_str_list.append('request_disk = %d' % (1024 * req_value)) # TODO: GLIDEIN_REQUIRE_GLEXEC_USE, WMS.SOFTWARE # (HPDA) file location service if 'dataFiles' in self._pool_req_dict: jdl_req_str_list.extend(self._get_dataset_fn_list(jobnum, task)) return jdl_req_str_list
def _format_reqs_sites(self, sites): def _fmt_sites(site): return 'RegExp(%s, other.GlueCEUniqueID)' % self._escape(site) (blacklist, whitelist) = split_blackwhite_list(sites) sitereqs = lmap(lambda x: '!' + _fmt_sites(x), blacklist) if whitelist: sitereqs.append('(%s)' % str.join(' || ', imap(_fmt_sites, whitelist))) if sitereqs: return '( %s )' % str.join(' && ', sitereqs)
def _getRequirementJdlData(self, task, jobnum): """Create JDL attributes corresponding to job requirements""" jdlData = [] requirements = task.get_requirement_list(jobnum) poolRequMap = self.parentPool.jdlRequirementMap for reqType, reqValue in requirements: # ('WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE') if reqType == WMS.SITES: (wantSites, vetoSites) = split_blackwhite_list(reqValue[1]) if "+SITES" in poolRequMap: jdlData.append( '%s = "%s"' % ( poolRequMap["+SITES"][0], poolRequMap["+SITES"][1] % ','.join(wantSites) ) ) if "-SITES" in poolRequMap: jdlData.append( '%s = "%s"' % ( poolRequMap["-SITES"][0], poolRequMap["-SITES"][1] % ','.join(vetoSites) ) ) continue if reqType == WMS.STORAGE: if ("STORAGE" in poolRequMap) and reqValue > 0: jdlData.append( '%s = %s ' % ( poolRequMap["STORAGE"][0], poolRequMap["STORAGE"][1] % ','.join(reqValue) ) ) continue #HACK if reqValue > 0 and WMS.reqTypes[reqType] in poolRequMap: jdlData.append( "%s = %s" % ( poolRequMap[WMS.reqTypes[reqType]][0], poolRequMap[WMS.reqTypes[reqType]][1] % reqValue ) ) continue try: if int(reqValue) <= 0: continue except TypeError: clear_current_exception() self._log(logging.INFO3, "Requirement '%s' cannot be mapped to pool and will be ignored!" % WMS.reqTypes[reqType]) return jdlData
def _getRequirementJdlData(self, task, jobnum): """Create JDL attributes corresponding to job requirements""" jdlData = [] requirements = task.get_requirement_list(jobnum) poolRequMap = self.parentPool.jdlRequirementMap for reqType, reqValue in requirements: # ('WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE') if reqType == WMS.SITES: (wantSites, vetoSites) = split_blackwhite_list(reqValue[1]) if "+SITES" in poolRequMap: jdlData.append( '%s = "%s"' % (poolRequMap["+SITES"][0], poolRequMap["+SITES"][1] % ','.join(wantSites))) if "-SITES" in poolRequMap: jdlData.append( '%s = "%s"' % (poolRequMap["-SITES"][0], poolRequMap["-SITES"][1] % ','.join(vetoSites))) continue if reqType == WMS.STORAGE: if ("STORAGE" in poolRequMap) and reqValue > 0: jdlData.append( '%s = %s ' % (poolRequMap["STORAGE"][0], poolRequMap["STORAGE"][1] % ','.join(reqValue))) continue #HACK if reqValue > 0 and WMS.reqTypes[reqType] in poolRequMap: jdlData.append( "%s = %s" % (poolRequMap[WMS.reqTypes[reqType]][0], poolRequMap[WMS.reqTypes[reqType]][1] % reqValue)) continue try: if int(reqValue) <= 0: continue except TypeError: clear_current_exception() self._log( logging.INFO3, "Requirement '%s' cannot be mapped to pool and will be ignored!" % WMS.reqTypes[reqType]) return jdlData