def crabCheck(self):
from CrabLogger import CrabLogger
from WorkSpace import WorkSpace, common
import tempfile, urllib, os, string
dname = tempfile.mkdtemp( "", "crab_", '/tmp' )
os.system("mkdir %s/log"%dname )
os.system("touch %s/crab.log"%dname )
cfg_params={'USER.logdir' : dname }
common.work_space = WorkSpace(dname, cfg_params)
args = string.join(sys.argv,' ')
common.debugLevel = 0
common.logger = CrabLogger(args)
from crab_util import getDN,gethnUserNameFromSiteDB
print 'start using CRAB utils ...\n'
print "my DN is: %s \n"%getDN()
try:
print 'my HN user name is: %s \n'%gethnUserNameFromSiteDB()
except:
print '\nWARNING native crab_utils failed! '
dn=urllib.urlencode({'dn':getDN()})
print 'trying now using urlencoded DN: \n\t %s '%dn
status,hnName = self.gethnName_urlenc(dn)
if status == 1:
print '\nWARNING: failed also using urlencoded DN '
else:
print 'my HN user name is: %s \n'%name
print 'problems with crab_utils'
print '\nend check.....................'
os.system("rm -rf %s"%dname )
def sched_parameter(self, i, task):
"""
Return scheduler-specific parameters. Used at crab -submit time
by $CRABPYTHON/Scheduler.py
"""
#SB paste from crab SchedulerGlidein
jobParams = ""
(self.remoteHost,self.remoteUserHost) = self.pickRemoteSubmissionHost(task)
psnDest = task.jobs[i-1]['dlsDestination']
if psnDest == [''] :
# crab.cfg had datasetpath = None
pnn2psn = getMapOfPhedexNodeName2ProcessingNodeNameFromSiteDB()
allPSNs = set(pnn2psn.values()) # set removes duplicates
psnDest = allPSNs
blackList = parseIntoList(self.cfg_params.get("GRID.se_black_list", []))
whiteList = parseIntoList(self.cfg_params.get("GRID.se_white_list", []))
psnDest = cleanPsnListForBlackWhiteLists(psnDest, blackList, whiteList)
if not psnDest or psnDest == [] or psnDest == ['']:
msg = "No Processing Site Name after applying black/white list."
msg += " Can't submit"
msg += "\nUsed Black List: %s" % blackList
msg += "\nUsed White List: %s" % whiteList
common.logger.info(msg)
raise CrabException(msg)
# condor's JDL needs a string of CSV's, not a list
psnDest = ",".join(psnDest)
msg = "list of PSN's for submission: %s" % psnDest
common.logger.info(msg)
jobParams += '+DESIRED_Sites = "%s";' % psnDest
scram = Scram.Scram(None)
cmsVersion = scram.getSWVersion()
scramArch = scram.getArch()
cmsver=re.split('_', cmsVersion)
if cmsver[3]=='DEVEL': cmsver[3]='0'
numericCmsVersion = "%s%.2d%.2d" %(cmsver[1], int(cmsver[2]), int(cmsver[3]))
if "slc5" in scramArch:
cmsOpSys = "LINUX"
cmsOpSysMajorVer = "5,6" # SL5 exe's also run on SL6
if "slc6" in scramArch:
cmsOpSys = "LINUX"
cmsOpSysMajorVer = "6"
if "ia32" in scramArch:
cmsArch = "INTEL,X86_64" # 32bit exe's also run on 64bit
if "amd64" in scramArch:
cmsArch = "X86_64"
# protect against datasetPath being None
jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";'
jobParams += '+DESIRED_CMSVersion ="' +cmsVersion+'";'
jobParams += '+DESIRED_CMSVersionNr ="' +numericCmsVersion+'";'
jobParams += '+DESIRED_CMSScramArch ="' +scramArch+'";'
jobParams += '+DESIRED_OpSyses ="' +cmsOpSys+'";'
jobParams += '+DESIRED_OpSysMajorVers ="' +cmsOpSysMajorVer+'";'
jobParams += '+DESIRED_Archs ="' +cmsArch+'";'
userName = gethnUserNameFromSiteDB()
jobParams += '+AccountingGroup = "analysis.%s";' % userName
myscheddName = self.remoteHost
jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \
'//' + self.submissionDay + '//$(Cluster).$(Process)"; '
if (self.EDG_clock_time):
glideinTime = "%d" % (int(self.EDG_clock_time)+20) # 20 min to wrapup
jobParams += '+MaxWallTimeMins = '+ glideinTime + '; '
else:
jobParams += '+MaxWallTimeMins = %d; ' % (21*60+55) # 21:55h (unit = min)
if self.max_rss :
jobParams += 'request_memory = '+self.max_rss+';'
if self.allowOverflow == "0":
jobParams += '+CMS_ALLOW_OVERFLOW = False; '
if self.EDG_addJdlParam:
if self.EDG_addJdlParam[-1] == '':
self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
for p in self.EDG_addJdlParam:
jobParams += p.strip()+';\n'
common._db.updateTask_({'jobType':jobParams})
return jobParams
def sched_parameter(self, i, task):
"""
Return scheduler-specific parameters. Used at crab -submit time
by $CRABPYTHON/Scheduler.py
"""
#SB paste from crab SchedulerGlidein
jobParams = ""
(self.remoteHost,self.remoteUserHost) = self.pickRemoteSubmissionHost(task)
seDest = task.jobs[i-1]['dlsDestination']
if seDest == [''] :
seDest = self.blackWhiteListParser.expandList("T") # all of SiteDB
seString=self.blackWhiteListParser.cleanForBlackWhiteList(seDest)
jobParams += '+DESIRED_SEs = "'+seString+'"; '
scram = Scram.Scram(None)
cmsVersion = scram.getSWVersion()
scramArch = scram.getArch()
cmsver=re.split('_', cmsVersion)
numericCmsVersion = "%s%.2d%.2d" %(cmsver[1], int(cmsver[2]), int(cmsver[3]))
jobParams += '+DESIRED_CMSVersion ="' +cmsVersion+'";'
jobParams += '+DESIRED_CMSVersionNr ="' +numericCmsVersion+'";'
jobParams += '+DESIRED_CMSScramArch ="' +scramArch+'";'
userName = gethnUserNameFromSiteDB()
jobParams += '+AccountingGroup ="' + userName+'";'
myscheddName = self.remoteHost
jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \
'//' + self.submissionDay + '//$(Cluster).$(Process)"; '
if (self.EDG_clock_time):
glideinTime = "%d" % (int(self.EDG_clock_time)+20) # 20 min to wrapup
jobParams += '+MaxWallTimeMins = '+ glideinTime + '; '
else:
jobParams += '+MaxWallTimeMins = %d; ' % (21*60+55) # 21:55h (unit = min)
if self.max_rss :
jobParams += 'request_memory = '+self.max_rss+';'
if self.allowOverflow == "0":
jobParams += '+CMS_ALLOW_OVERFLOW = False; '
if self.EDG_addJdlParam:
if self.EDG_addJdlParam[-1] == '':
self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
for p in self.EDG_addJdlParam:
jobParams += p.strip()+';\n'
common._db.updateTask_({'jobType':jobParams})
return jobParams
def sched_parameter(self, i, task):
"""
Return scheduler-specific parameters. Used at crab -submit time
by $CRABPYTHON/Scheduler.py
"""
#SB paste from crab SchedulerGlidein
jobParams = ""
(self.remoteHost,self.remoteUserHost) = self.pickRemoteSubmissionHost(task)
seDest = task.jobs[i-1]['dlsDestination']
if seDest == [''] :
seDest = self.blackWhiteListParser.expandList("T") # all of SiteDB
seString=self.blackWhiteListParser.cleanForBlackWhiteList(seDest)
# beware SiteDB V2 API, explicely cast to string in case it is unicode
seString=str(seString)
jobParams += '+DESIRED_SEs = "'+seString+'"; '
scram = Scram.Scram(None)
cmsVersion = scram.getSWVersion()
scramArch = scram.getArch()
cmsver=re.split('_', cmsVersion)
numericCmsVersion = "%s%.2d%.2d" %(cmsver[1], int(cmsver[2]), int(cmsver[3]))
if "slc5" in scramArch:
cmsOpSys = "LINUX"
cmsOpSysMajorVer = "5,6" # SL5 exe's also run on SL6
if "slc6" in scramArch:
cmsOpSys = "LINUX"
cmsOpSysMajorVer = "6"
if "ia32" in scramArch:
cmsArch = "INTEL,X86_64" # 32bit exe's also run on 64bit
if "amd64" in scramArch:
cmsArch = "X86_64"
# protect against datasetPath being None
jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";'
jobParams += '+DESIRED_CMSVersion ="' +cmsVersion+'";'
jobParams += '+DESIRED_CMSVersionNr ="' +numericCmsVersion+'";'
jobParams += '+DESIRED_CMSScramArch ="' +scramArch+'";'
jobParams += '+DESIRED_OpSyses ="' +cmsOpSys+'";'
jobParams += '+DESIRED_OpSysMajorVers ="' +cmsOpSysMajorVer+'";'
jobParams += '+DESIRED_Archs ="' +cmsArch+'";'
userName = gethnUserNameFromSiteDB()
jobParams += '+AccountingGroup ="' + userName+'";'
myscheddName = self.remoteHost
jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \
'//' + self.submissionDay + '//$(Cluster).$(Process)"; '
if (self.EDG_clock_time):
glideinTime = "%d" % (int(self.EDG_clock_time)+20) # 20 min to wrapup
jobParams += '+MaxWallTimeMins = '+ glideinTime + '; '
else:
jobParams += '+MaxWallTimeMins = %d; ' % (21*60+55) # 21:55h (unit = min)
if self.max_rss :
jobParams += 'request_memory = '+self.max_rss+';'
if self.allowOverflow == "0":
jobParams += '+CMS_ALLOW_OVERFLOW = False; '
if self.EDG_addJdlParam:
if self.EDG_addJdlParam[-1] == '':
self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
for p in self.EDG_addJdlParam:
jobParams += p.strip()+';\n'
common._db.updateTask_({'jobType':jobParams})
return jobParams
def sched_parameter(self, i, task):
"""
Return scheduler-specific parameters. Used at crab -submit time
by $CRABPYTHON/Scheduler.py
"""
#SB paste from crab SchedulerGlidein
jobParams = ""
(self.remoteHost,
self.remoteUserHost) = self.pickRemoteSubmissionHost(task)
seDest = task.jobs[i - 1]['dlsDestination']
if seDest == ['']:
seDest = self.blackWhiteListParser.expandList("T") # all of SiteDB
seString = self.blackWhiteListParser.cleanForBlackWhiteList(seDest)
# beware SiteDB V2 API, explicely cast to string in case it is unicode
seString = str(seString)
jobParams += '+DESIRED_SEs = "' + seString + '"; '
scram = Scram.Scram(None)
cmsVersion = scram.getSWVersion()
scramArch = scram.getArch()
cmsver = re.split('_', cmsVersion)
numericCmsVersion = "%s%.2d%.2d" % (cmsver[1], int(
cmsver[2]), int(cmsver[3]))
# protect against datasetPath being None
jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";'
jobParams += '+DESIRED_CMSVersion ="' + cmsVersion + '";'
jobParams += '+DESIRED_CMSVersionNr ="' + numericCmsVersion + '";'
jobParams += '+DESIRED_CMSScramArch ="' + scramArch + '";'
userName = gethnUserNameFromSiteDB()
jobParams += '+AccountingGroup ="' + userName + '";'
myscheddName = self.remoteHost
jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \
'//' + self.submissionDay + '//$(Cluster).$(Process)"; '
if (self.EDG_clock_time):
glideinTime = "%d" % (int(self.EDG_clock_time) + 20
) # 20 min to wrapup
jobParams += '+MaxWallTimeMins = ' + glideinTime + '; '
else:
jobParams += '+MaxWallTimeMins = %d; ' % (
21 * 60 + 55) # 21:55h (unit = min)
if self.max_rss:
jobParams += 'request_memory = ' + self.max_rss + ';'
if self.allowOverflow == "0":
jobParams += '+CMS_ALLOW_OVERFLOW = False; '
if self.EDG_addJdlParam:
if self.EDG_addJdlParam[-1] == '':
self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
for p in self.EDG_addJdlParam:
jobParams += p.strip() + ';\n'
common._db.updateTask_({'jobType': jobParams})
return jobParams
def sched_parameter(self, i, task):
"""
Return scheduler-specific parameters. Used at crab -submit time
by $CRABPYTHON/Scheduler.py
"""
#SB paste from crab SchedulerGlidein
jobParams = ""
(self.remoteHost,
self.remoteUserHost) = self.pickRemoteSubmissionHost(task)
psnDest = task.jobs[i - 1]['dlsDestination']
if psnDest == ['']:
# crab.cfg had datasetpath = None
pnn2psn = getMapOfPhedexNodeName2ProcessingNodeNameFromSiteDB()
allPSNs = set(pnn2psn.values()) # set removes duplicates
psnDest = allPSNs
blackList = parseIntoList(self.cfg_params.get("GRID.se_black_list",
[]))
whiteList = parseIntoList(self.cfg_params.get("GRID.se_white_list",
[]))
psnDest = cleanPsnListForBlackWhiteLists(psnDest, blackList, whiteList)
if not psnDest or psnDest == [] or psnDest == ['']:
msg = "No Processing Site Name after applying black/white list."
msg += " Can't submit"
msg += "\nUsed Black List: %s" % blackList
msg += "\nUsed White List: %s" % whiteList
common.logger.info(msg)
raise CrabException(msg)
# condor's JDL needs a string of CSV's, not a list
psnDest = ",".join(psnDest)
msg = "list of PSN's for submission: %s" % psnDest
common.logger.info(msg)
jobParams += '+DESIRED_Sites = "%s";' % psnDest
scram = Scram.Scram(None)
cmsVersion = scram.getSWVersion()
scramArch = scram.getArch()
cmsver = re.split('_', cmsVersion)
if cmsver[3] == 'DEVEL': cmsver[3] = '0'
numericCmsVersion = "%s%.2d%.2d" % (cmsver[1], int(
cmsver[2]), int(cmsver[3]))
if "slc5" in scramArch:
cmsOpSys = "LINUX"
cmsOpSysMajorVer = "5,6" # SL5 exe's also run on SL6
if "slc6" in scramArch:
cmsOpSys = "LINUX"
cmsOpSysMajorVer = "6"
if "ia32" in scramArch:
cmsArch = "INTEL,X86_64" # 32bit exe's also run on 64bit
if "amd64" in scramArch:
cmsArch = "X86_64"
# protect against datasetPath being None
jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";'
jobParams += '+DESIRED_CMSVersion ="' + cmsVersion + '";'
jobParams += '+DESIRED_CMSVersionNr ="' + numericCmsVersion + '";'
jobParams += '+DESIRED_CMSScramArch ="' + scramArch + '";'
jobParams += '+DESIRED_OpSyses ="' + cmsOpSys + '";'
jobParams += '+DESIRED_OpSysMajorVers ="' + cmsOpSysMajorVer + '";'
jobParams += '+DESIRED_Archs ="' + cmsArch + '";'
userName = gethnUserNameFromSiteDB()
jobParams += '+AccountingGroup = "analysis.%s";' % userName
myscheddName = self.remoteHost
jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \
'//' + self.submissionDay + '//$(Cluster).$(Process)"; '
if (self.EDG_clock_time):
glideinTime = "%d" % (int(self.EDG_clock_time) + 20
) # 20 min to wrapup
jobParams += '+MaxWallTimeMins = ' + glideinTime + '; '
else:
jobParams += '+MaxWallTimeMins = %d; ' % (
21 * 60 + 55) # 21:55h (unit = min)
if self.max_rss:
jobParams += 'request_memory = ' + self.max_rss + ';'
if self.allowOverflow == "0":
jobParams += '+CMS_ALLOW_OVERFLOW = False; '
if self.EDG_addJdlParam:
if self.EDG_addJdlParam[-1] == '':
self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
for p in self.EDG_addJdlParam:
jobParams += p.strip() + ';\n'
common._db.updateTask_({'jobType': jobParams})
return jobParams
