def crabCheck(self): from CrabLogger import CrabLogger from WorkSpace import WorkSpace, common import tempfile, urllib, os, string dname = tempfile.mkdtemp( "", "crab_", '/tmp' ) os.system("mkdir %s/log"%dname ) os.system("touch %s/crab.log"%dname ) cfg_params={'USER.logdir' : dname } common.work_space = WorkSpace(dname, cfg_params) args = string.join(sys.argv,' ') common.debugLevel = 0 common.logger = CrabLogger(args) from crab_util import getDN,gethnUserNameFromSiteDB print 'start using CRAB utils ...\n' print "my DN is: %s \n"%getDN() try: print 'my HN user name is: %s \n'%gethnUserNameFromSiteDB() except: print '\nWARNING native crab_utils failed! ' dn=urllib.urlencode({'dn':getDN()}) print 'trying now using urlencoded DN: \n\t %s '%dn status,hnName = self.gethnName_urlenc(dn) if status == 1: print '\nWARNING: failed also using urlencoded DN ' else: print 'my HN user name is: %s \n'%name print 'problems with crab_utils' print '\nend check.....................' os.system("rm -rf %s"%dname )
def sched_parameter(self, i, task): """ Return scheduler-specific parameters. Used at crab -submit time by $CRABPYTHON/Scheduler.py """ #SB paste from crab SchedulerGlidein jobParams = "" (self.remoteHost,self.remoteUserHost) = self.pickRemoteSubmissionHost(task) psnDest = task.jobs[i-1]['dlsDestination'] if psnDest == [''] : # crab.cfg had datasetpath = None pnn2psn = getMapOfPhedexNodeName2ProcessingNodeNameFromSiteDB() allPSNs = set(pnn2psn.values()) # set removes duplicates psnDest = allPSNs blackList = parseIntoList(self.cfg_params.get("GRID.se_black_list", [])) whiteList = parseIntoList(self.cfg_params.get("GRID.se_white_list", [])) psnDest = cleanPsnListForBlackWhiteLists(psnDest, blackList, whiteList) if not psnDest or psnDest == [] or psnDest == ['']: msg = "No Processing Site Name after applying black/white list." msg += " Can't submit" msg += "\nUsed Black List: %s" % blackList msg += "\nUsed White List: %s" % whiteList common.logger.info(msg) raise CrabException(msg) # condor's JDL needs a string of CSV's, not a list psnDest = ",".join(psnDest) msg = "list of PSN's for submission: %s" % psnDest common.logger.info(msg) jobParams += '+DESIRED_Sites = "%s";' % psnDest scram = Scram.Scram(None) cmsVersion = scram.getSWVersion() scramArch = scram.getArch() cmsver=re.split('_', cmsVersion) if cmsver[3]=='DEVEL': cmsver[3]='0' numericCmsVersion = "%s%.2d%.2d" %(cmsver[1], int(cmsver[2]), int(cmsver[3])) if "slc5" in scramArch: cmsOpSys = "LINUX" cmsOpSysMajorVer = "5,6" # SL5 exe's also run on SL6 if "slc6" in scramArch: cmsOpSys = "LINUX" cmsOpSysMajorVer = "6" if "ia32" in scramArch: cmsArch = "INTEL,X86_64" # 32bit exe's also run on 64bit if "amd64" in scramArch: cmsArch = "X86_64" # protect against datasetPath being None jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";' jobParams += '+DESIRED_CMSVersion ="' +cmsVersion+'";' jobParams += '+DESIRED_CMSVersionNr ="' +numericCmsVersion+'";' jobParams += '+DESIRED_CMSScramArch ="' +scramArch+'";' jobParams += '+DESIRED_OpSyses ="' +cmsOpSys+'";' jobParams += '+DESIRED_OpSysMajorVers ="' +cmsOpSysMajorVer+'";' jobParams += '+DESIRED_Archs ="' +cmsArch+'";' userName = gethnUserNameFromSiteDB() jobParams += '+AccountingGroup = "analysis.%s";' % userName myscheddName = self.remoteHost jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \ '//' + self.submissionDay + '//$(Cluster).$(Process)"; ' if (self.EDG_clock_time): glideinTime = "%d" % (int(self.EDG_clock_time)+20) # 20 min to wrapup jobParams += '+MaxWallTimeMins = '+ glideinTime + '; ' else: jobParams += '+MaxWallTimeMins = %d; ' % (21*60+55) # 21:55h (unit = min) if self.max_rss : jobParams += 'request_memory = '+self.max_rss+';' if self.allowOverflow == "0": jobParams += '+CMS_ALLOW_OVERFLOW = False; ' if self.EDG_addJdlParam: if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam = self.EDG_addJdlParam[:-1] for p in self.EDG_addJdlParam: jobParams += p.strip()+';\n' common._db.updateTask_({'jobType':jobParams}) return jobParams
def sched_parameter(self, i, task): """ Return scheduler-specific parameters. Used at crab -submit time by $CRABPYTHON/Scheduler.py """ #SB paste from crab SchedulerGlidein jobParams = "" (self.remoteHost,self.remoteUserHost) = self.pickRemoteSubmissionHost(task) seDest = task.jobs[i-1]['dlsDestination'] if seDest == [''] : seDest = self.blackWhiteListParser.expandList("T") # all of SiteDB seString=self.blackWhiteListParser.cleanForBlackWhiteList(seDest) jobParams += '+DESIRED_SEs = "'+seString+'"; ' scram = Scram.Scram(None) cmsVersion = scram.getSWVersion() scramArch = scram.getArch() cmsver=re.split('_', cmsVersion) numericCmsVersion = "%s%.2d%.2d" %(cmsver[1], int(cmsver[2]), int(cmsver[3])) jobParams += '+DESIRED_CMSVersion ="' +cmsVersion+'";' jobParams += '+DESIRED_CMSVersionNr ="' +numericCmsVersion+'";' jobParams += '+DESIRED_CMSScramArch ="' +scramArch+'";' userName = gethnUserNameFromSiteDB() jobParams += '+AccountingGroup ="' + userName+'";' myscheddName = self.remoteHost jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \ '//' + self.submissionDay + '//$(Cluster).$(Process)"; ' if (self.EDG_clock_time): glideinTime = "%d" % (int(self.EDG_clock_time)+20) # 20 min to wrapup jobParams += '+MaxWallTimeMins = '+ glideinTime + '; ' else: jobParams += '+MaxWallTimeMins = %d; ' % (21*60+55) # 21:55h (unit = min) if self.max_rss : jobParams += 'request_memory = '+self.max_rss+';' if self.allowOverflow == "0": jobParams += '+CMS_ALLOW_OVERFLOW = False; ' if self.EDG_addJdlParam: if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam = self.EDG_addJdlParam[:-1] for p in self.EDG_addJdlParam: jobParams += p.strip()+';\n' common._db.updateTask_({'jobType':jobParams}) return jobParams
def sched_parameter(self, i, task): """ Return scheduler-specific parameters. Used at crab -submit time by $CRABPYTHON/Scheduler.py """ #SB paste from crab SchedulerGlidein jobParams = "" (self.remoteHost,self.remoteUserHost) = self.pickRemoteSubmissionHost(task) seDest = task.jobs[i-1]['dlsDestination'] if seDest == [''] : seDest = self.blackWhiteListParser.expandList("T") # all of SiteDB seString=self.blackWhiteListParser.cleanForBlackWhiteList(seDest) # beware SiteDB V2 API, explicely cast to string in case it is unicode seString=str(seString) jobParams += '+DESIRED_SEs = "'+seString+'"; ' scram = Scram.Scram(None) cmsVersion = scram.getSWVersion() scramArch = scram.getArch() cmsver=re.split('_', cmsVersion) numericCmsVersion = "%s%.2d%.2d" %(cmsver[1], int(cmsver[2]), int(cmsver[3])) if "slc5" in scramArch: cmsOpSys = "LINUX" cmsOpSysMajorVer = "5,6" # SL5 exe's also run on SL6 if "slc6" in scramArch: cmsOpSys = "LINUX" cmsOpSysMajorVer = "6" if "ia32" in scramArch: cmsArch = "INTEL,X86_64" # 32bit exe's also run on 64bit if "amd64" in scramArch: cmsArch = "X86_64" # protect against datasetPath being None jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";' jobParams += '+DESIRED_CMSVersion ="' +cmsVersion+'";' jobParams += '+DESIRED_CMSVersionNr ="' +numericCmsVersion+'";' jobParams += '+DESIRED_CMSScramArch ="' +scramArch+'";' jobParams += '+DESIRED_OpSyses ="' +cmsOpSys+'";' jobParams += '+DESIRED_OpSysMajorVers ="' +cmsOpSysMajorVer+'";' jobParams += '+DESIRED_Archs ="' +cmsArch+'";' userName = gethnUserNameFromSiteDB() jobParams += '+AccountingGroup ="' + userName+'";' myscheddName = self.remoteHost jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \ '//' + self.submissionDay + '//$(Cluster).$(Process)"; ' if (self.EDG_clock_time): glideinTime = "%d" % (int(self.EDG_clock_time)+20) # 20 min to wrapup jobParams += '+MaxWallTimeMins = '+ glideinTime + '; ' else: jobParams += '+MaxWallTimeMins = %d; ' % (21*60+55) # 21:55h (unit = min) if self.max_rss : jobParams += 'request_memory = '+self.max_rss+';' if self.allowOverflow == "0": jobParams += '+CMS_ALLOW_OVERFLOW = False; ' if self.EDG_addJdlParam: if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam = self.EDG_addJdlParam[:-1] for p in self.EDG_addJdlParam: jobParams += p.strip()+';\n' common._db.updateTask_({'jobType':jobParams}) return jobParams
def sched_parameter(self, i, task): """ Return scheduler-specific parameters. Used at crab -submit time by $CRABPYTHON/Scheduler.py """ #SB paste from crab SchedulerGlidein jobParams = "" (self.remoteHost, self.remoteUserHost) = self.pickRemoteSubmissionHost(task) seDest = task.jobs[i - 1]['dlsDestination'] if seDest == ['']: seDest = self.blackWhiteListParser.expandList("T") # all of SiteDB seString = self.blackWhiteListParser.cleanForBlackWhiteList(seDest) # beware SiteDB V2 API, explicely cast to string in case it is unicode seString = str(seString) jobParams += '+DESIRED_SEs = "' + seString + '"; ' scram = Scram.Scram(None) cmsVersion = scram.getSWVersion() scramArch = scram.getArch() cmsver = re.split('_', cmsVersion) numericCmsVersion = "%s%.2d%.2d" % (cmsver[1], int( cmsver[2]), int(cmsver[3])) # protect against datasetPath being None jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";' jobParams += '+DESIRED_CMSVersion ="' + cmsVersion + '";' jobParams += '+DESIRED_CMSVersionNr ="' + numericCmsVersion + '";' jobParams += '+DESIRED_CMSScramArch ="' + scramArch + '";' userName = gethnUserNameFromSiteDB() jobParams += '+AccountingGroup ="' + userName + '";' myscheddName = self.remoteHost jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \ '//' + self.submissionDay + '//$(Cluster).$(Process)"; ' if (self.EDG_clock_time): glideinTime = "%d" % (int(self.EDG_clock_time) + 20 ) # 20 min to wrapup jobParams += '+MaxWallTimeMins = ' + glideinTime + '; ' else: jobParams += '+MaxWallTimeMins = %d; ' % ( 21 * 60 + 55) # 21:55h (unit = min) if self.max_rss: jobParams += 'request_memory = ' + self.max_rss + ';' if self.allowOverflow == "0": jobParams += '+CMS_ALLOW_OVERFLOW = False; ' if self.EDG_addJdlParam: if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam = self.EDG_addJdlParam[:-1] for p in self.EDG_addJdlParam: jobParams += p.strip() + ';\n' common._db.updateTask_({'jobType': jobParams}) return jobParams
def sched_parameter(self, i, task): """ Return scheduler-specific parameters. Used at crab -submit time by $CRABPYTHON/Scheduler.py """ #SB paste from crab SchedulerGlidein jobParams = "" (self.remoteHost, self.remoteUserHost) = self.pickRemoteSubmissionHost(task) psnDest = task.jobs[i - 1]['dlsDestination'] if psnDest == ['']: # crab.cfg had datasetpath = None pnn2psn = getMapOfPhedexNodeName2ProcessingNodeNameFromSiteDB() allPSNs = set(pnn2psn.values()) # set removes duplicates psnDest = allPSNs blackList = parseIntoList(self.cfg_params.get("GRID.se_black_list", [])) whiteList = parseIntoList(self.cfg_params.get("GRID.se_white_list", [])) psnDest = cleanPsnListForBlackWhiteLists(psnDest, blackList, whiteList) if not psnDest or psnDest == [] or psnDest == ['']: msg = "No Processing Site Name after applying black/white list." msg += " Can't submit" msg += "\nUsed Black List: %s" % blackList msg += "\nUsed White List: %s" % whiteList common.logger.info(msg) raise CrabException(msg) # condor's JDL needs a string of CSV's, not a list psnDest = ",".join(psnDest) msg = "list of PSN's for submission: %s" % psnDest common.logger.info(msg) jobParams += '+DESIRED_Sites = "%s";' % psnDest scram = Scram.Scram(None) cmsVersion = scram.getSWVersion() scramArch = scram.getArch() cmsver = re.split('_', cmsVersion) if cmsver[3] == 'DEVEL': cmsver[3] = '0' numericCmsVersion = "%s%.2d%.2d" % (cmsver[1], int( cmsver[2]), int(cmsver[3])) if "slc5" in scramArch: cmsOpSys = "LINUX" cmsOpSysMajorVer = "5,6" # SL5 exe's also run on SL6 if "slc6" in scramArch: cmsOpSys = "LINUX" cmsOpSysMajorVer = "6" if "ia32" in scramArch: cmsArch = "INTEL,X86_64" # 32bit exe's also run on 64bit if "amd64" in scramArch: cmsArch = "X86_64" # protect against datasetPath being None jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";' jobParams += '+DESIRED_CMSVersion ="' + cmsVersion + '";' jobParams += '+DESIRED_CMSVersionNr ="' + numericCmsVersion + '";' jobParams += '+DESIRED_CMSScramArch ="' + scramArch + '";' jobParams += '+DESIRED_OpSyses ="' + cmsOpSys + '";' jobParams += '+DESIRED_OpSysMajorVers ="' + cmsOpSysMajorVer + '";' jobParams += '+DESIRED_Archs ="' + cmsArch + '";' userName = gethnUserNameFromSiteDB() jobParams += '+AccountingGroup = "analysis.%s";' % userName myscheddName = self.remoteHost jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \ '//' + self.submissionDay + '//$(Cluster).$(Process)"; ' if (self.EDG_clock_time): glideinTime = "%d" % (int(self.EDG_clock_time) + 20 ) # 20 min to wrapup jobParams += '+MaxWallTimeMins = ' + glideinTime + '; ' else: jobParams += '+MaxWallTimeMins = %d; ' % ( 21 * 60 + 55) # 21:55h (unit = min) if self.max_rss: jobParams += 'request_memory = ' + self.max_rss + ';' if self.allowOverflow == "0": jobParams += '+CMS_ALLOW_OVERFLOW = False; ' if self.EDG_addJdlParam: if self.EDG_addJdlParam[-1] == '': self.EDG_addJdlParam = self.EDG_addJdlParam[:-1] for p in self.EDG_addJdlParam: jobParams += p.strip() + ';\n' common._db.updateTask_({'jobType': jobParams}) return jobParams