示例#1
0
    def __init__(self, cfg_params, parsed_range, val):
        self.cfg_params = cfg_params
        self.limitJobs = True
        # get user request
        self.nsjobs = -1
        self.chosenJobsList = None
        if val:
            if val=='range':  # for Resubmitter
                self.chosenJobsList = parsed_range
            elif val=='all':
                pass
            elif (type(eval(val)) is int) and eval(val) > 0:
                # positive number
                self.nsjobs = eval(val)
            elif (type(eval(val)) is tuple)or( type(eval(val)) is int and eval(val)<0 ) :
                self.chosenJobsList = parsed_range
                self.nsjobs = len(self.chosenJobsList)
            else:
                msg = 'Bad submission option <'+str(val)+'>\n'
                msg += '      Must be an integer or "all"'
                msg += '      Generic range is not allowed"'
                raise CrabException(msg)
            pass

        self.seWhiteList = parseIntoList(cfg_params.get('GRID.se_white_list',[]))
        self.seBlackList = parseIntoList(cfg_params.get('GRID.se_black_list',[]))
        self.datasetPath = cfg_params['CMSSW.datasetpath']
        if string.lower(self.datasetPath)=='none':
            self.datasetPath = None
        self.scram = Scram.Scram(cfg_params)
        return
示例#2
0
class ServerCommunicator:
    """
    Common interface for the interaction between the Crab client and the server Web Service
    """
    def __init__(self, serverName, serverPort, cfg_params, proxyPath=None):
        """
        Open the communication with an Analysis Server by passing the server URL and the port
        """

        self.ServerTwiki = 'https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideCrabServerForUsers#Server_available_for_users'

        self.asSession = C_AS_Session(serverName, serverPort)
        self.cfg_params = cfg_params
        self.userSubj = ''
        self.serverName = serverName
        credentialType = 'Proxy'
        if common.scheduler.name().upper() in ['CAF','LSF']:
            credentialType = 'Token'
        CliServerParams(self)
        self.crab_task_name = common.work_space.topDir().split('/')[-2] # nice task name "crab_0_..."

        configAPI = {'credential' : credentialType, \
                     'logger' : common.logger() }

        CredAPI =  CredentialAPI( configAPI )
        try:
            self.userSubj = CredAPI.getSubject()
        except Exception, err:
            common.logger.debug("Getting Credential Subject: " +str(traceback.format_exc()))
            raise CrabException("Error Getting Credential Subject")

        self.scram=Scram.Scram(cfg_params)
示例#3
0
    def __init__(self, cfg_params, template):
        # pattern -> action
        self.actions = {
            'title': self.title_,
            'untar_software': self.untarSoftware_,
            'initial_environment': self.initialEnvironment_,
            'setup_scheduler_environment': self.setupSchedulerEnvironment_,
            'setup_jobtype_environment': self.setupJobTypeEnvironment_,
            'copy_input': self.copyInput_,
            'rewrite_cmssw_cfg': self.rewriteCMSSWcfg_,
            'build_executable': self.buildExe_,
            'run_executable': self.runExe_,
            'rename_output': self.renameOutput_,
            'copy_output': self.copyOutput_,
            'parse_report': self.parseReport_,
            'modify_report': self.modifyReport_,
            'func_exit': self.func_exit_
        }

        if os.path.isfile("./" + template):
            self.template = "./" + template
        elif os.getenv('CRABDIR') and os.path.isfile(
                os.getenv('CRABDIR') + '/python/' + template):
            self.template = os.getenv('CRABDIR') + '/python/' + template
        else:
            raise CrabException("No crab_template.sh found!")
        self.nj = -1  # current job number

        try:
            self.scram = Scram.Scram(None)
            self.CMSSWversion = self.scram.getSWVersion()
            parts = self.CMSSWversion.split('_')
            self.CMSSW_major = int(parts[1])
            self.CMSSW_minor = int(parts[2])
            if parts[3] == 'DEVEL': parts[3] = '0'
            self.CMSSW_patch = int(parts[3])
        except:
            raise CrabException("Could not determine CMSSW version")
        self.debug_wrapper = ''
        debug = int(cfg_params.get('USER.debug_wrapper', 0))
        if debug == 1: self.debug_wrapper = '--debug'

        self.scriptName = cfg_params.get('CRAB.jobtype').upper() + '.sh'

        self.pset = cfg_params.get('CMSSW.pset', 'None')

        return
示例#4
0
    def sched_parameter(self, i, task):
        """
        Return scheduler-specific parameters. Used at crab -submit time
        by $CRABPYTHON/Scheduler.py
        """

        #SB paste from crab ScheduerGlidein

        jobParams = ""

        (self.rcondorHost,
         self.rcondorUserHost) = self.pickRcondorSubmissionHost(task)

        seDest = task.jobs[i - 1]['dlsDestination']

        if seDest == ['']:
            seDest = self.blackWhiteListParser.expandList("T")  # all of SiteDB

        seString = self.blackWhiteListParser.cleanForBlackWhiteList(seDest)

        jobParams += '+DESIRED_SEs = "' + seString + '"; '

        scram = Scram.Scram(None)
        cmsVersion = scram.getSWVersion()
        scramArch = scram.getArch()

        cmsver = re.split('_', cmsVersion)
        numericCmsVersion = "%s%.2d%.2d" % (cmsver[1], int(
            cmsver[2]), int(cmsver[3]))

        jobParams += '+DESIRED_CMSVersion ="' + cmsVersion + '";'
        jobParams += '+DESIRED_CMSVersionNr ="' + numericCmsVersion + '";'
        jobParams += '+DESIRED_CMSScramArch ="' + scramArch + '";'

        myscheddName = self.rcondorHost
        jobParams += '+Glidein_MonitorID = "https://' + myscheddName + '//$(Cluster).$(Process)"; '

        if (self.EDG_clock_time):
            jobParams += '+MaxWallTimeMins = ' + self.EDG_clock_time + '; '
        else:
            jobParams += '+MaxWallTimeMins = %d; ' % (60 * 24)

        common._db.updateTask_({'jobType': jobParams})

        return jobParams
示例#5
0
    def listMatch(self, seList, full, onlyOSG=True):
        """
        Check the compatibility of available resources
        """

        scram = Scram.Scram(None)
        versionCMSSW = scram.getSWVersion()
        arch = scram.getArch()

        if self.selectNoInput:
            availCEs = listAllCEs(versionCMSSW, arch, onlyOSG=onlyOSG)
        else:
            seDest = self.blackWhiteListParser.cleanForBlackWhiteList(seList, "list")
            availCEs = getJobManagerList(seDest, versionCMSSW,
                                         arch, onlyOSG=onlyOSG)

        uniqCEs = []
        for ce in availCEs:
            if ce not in uniqCEs:
                uniqCEs.append(ce)

        ceDest = self.ceBlackWhiteListParser.cleanForBlackWhiteList(uniqCEs, "list")

        return ceDest
示例#6
0
    def __init__(self, cfg_params, ncjobs):

        # parameter
        self.cfg_params = cfg_params
        self.ncjobs = ncjobs

        # dataset
        try:
            self.Dataset = cfg_params['SKIM.dataset']
        except:
            msg = 'Please specify a dataset'
            raise SkimException(msg)

        # pset
        try:
            self.pset = cfg_params['SKIM.pset']
            if (not os.path.exists(self.pset)):
                raise SkimException("User defined PSet file " + self.pset +
                                    " does not exist")
        except KeyError:
            raise SkimException("PSet file missing. Cannot run cmsRun ")

        # scram
        self.scram = Scram.Scram(cfg_params)
        scramArea = ''
        self.additional_input_files = []
        self.version = self.scram.getSWVersion()

        # name of tarball
        self.tgz_name = 'default.tgz'

        # additional input files
        try:
            tmpAddFiles = string.split(
                cfg_params['SKIM.additional_input_files'], ',')
            for tmp in tmpAddFiles:
                if not os.path.exists(tmp):
                    raise SkimException("Additional input file not found: " +
                                        tmp)
                self.additional_input_files.append(string.strip(tmp))
                pass
            pass
        except KeyError:
            pass

        # files per job
        try:
            if (cfg_params['SKIM.files_per_jobs']):
                self.filesPerJob = int(cfg_params['SKIM.files_per_job'])
                self.selectFilesPerJob = 1
        except KeyError:
            self.filesPerJob = -1
            self.selectFilesPerJob = 0

        # Events per job
        try:
            self.eventsPerJob = int(cfg_params['SKIM.events_per_job'])
            self.selectEventsPerJob = 1
        except KeyError:
            self.eventsPerJob = -1
            self.selectEventsPerJob = 0

        # number of jobs
        try:
            self.theNumberOfJobs = int(cfg_params['SKIM.number_of_jobs'])
            self.selectNumberOfJobs = 1
        except KeyError:
            self.theNumberOfJobs = 0
            self.selectNumberOfJobs = 0

        # total number of events
        try:
            self.total_number_of_events = int(
                cfg_params['SKIM.total_number_of_events'])
            self.selectTotalNumberEvents = 1
        except KeyError:
            self.total_number_of_events = 0
            self.selectTotalNumberEvents = 0

        # total number of files
        try:
            self.total_number_of_files = int(
                cfg_params['SKIM.total_number_of_files'])
            self.selectTotalNumberFiles = 1
        except KeyError:
            self.total_number_of_files = 0
            self.selectTotalNumberFiles = 0

        if (self.selectFilesPerJob + self.selectEventsPerJob == 2):
            msg = 'Must define either files per job or events per job.'
            raise SkimException(msg)

        if (self.selectEventsPerJob):
            if ((self.selectTotalNumberEvents + self.selectEventsPerJob +
                 self.selectNumberOfJobs) != 2):
                msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
                raise SkimException(msg)

        if (self.selectFilesPerJob):
            if ((self.selectTotalNumberFiles + self.selectNumberOfJobs +
                 self.selectFilesPerJob) != 2):
                msg = 'Must define exactly two of total_number_of_events, events_per_job, or number_of_jobs.'
                raise SkimException(msg)
        return
    def sched_parameter(self, i, task):
        """
        Return scheduler-specific parameters. Used at crab -submit time
        by $CRABPYTHON/Scheduler.py
        """

        #SB paste from crab SchedulerGlidein

        jobParams = ""

        (self.remoteHost,
         self.remoteUserHost) = self.pickRemoteSubmissionHost(task)

        seDest = task.jobs[i - 1]['dlsDestination']

        if seDest == ['']:
            seDest = self.blackWhiteListParser.expandList("T")  # all of SiteDB

        seString = self.blackWhiteListParser.cleanForBlackWhiteList(seDest)
        # beware SiteDB V2 API, explicely cast to string in case it is unicode
        seString = str(seString)

        jobParams += '+DESIRED_SEs = "' + seString + '"; '

        scram = Scram.Scram(None)
        cmsVersion = scram.getSWVersion()
        scramArch = scram.getArch()

        cmsver = re.split('_', cmsVersion)
        numericCmsVersion = "%s%.2d%.2d" % (cmsver[1], int(
            cmsver[2]), int(cmsver[3]))

        # protect against datasetPath being None
        jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";'

        jobParams += '+DESIRED_CMSVersion ="' + cmsVersion + '";'
        jobParams += '+DESIRED_CMSVersionNr ="' + numericCmsVersion + '";'
        jobParams += '+DESIRED_CMSScramArch ="' + scramArch + '";'

        userName = gethnUserNameFromSiteDB()
        jobParams += '+AccountingGroup ="' + userName + '";'

        myscheddName = self.remoteHost

        jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \
                     '//' + self.submissionDay + '//$(Cluster).$(Process)"; '

        if (self.EDG_clock_time):
            glideinTime = "%d" % (int(self.EDG_clock_time) + 20
                                  )  # 20 min to wrapup
            jobParams += '+MaxWallTimeMins = ' + glideinTime + '; '
        else:
            jobParams += '+MaxWallTimeMins = %d; ' % (
                21 * 60 + 55)  #  21:55h  (unit = min)

        if self.max_rss:
            jobParams += 'request_memory = ' + self.max_rss + ';'

        if self.allowOverflow == "0":
            jobParams += '+CMS_ALLOW_OVERFLOW = False; '

        if self.EDG_addJdlParam:
            if self.EDG_addJdlParam[-1] == '':
                self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
            for p in self.EDG_addJdlParam:
                jobParams += p.strip() + ';\n'

        common._db.updateTask_({'jobType': jobParams})

        return jobParams
示例#8
0
    def sched_parameter(self, i, task):
        """
        Return scheduler-specific parameters. Used at crab -submit time
        by $CRABPYTHON/Scheduler.py
        """

        #SB paste from crab SchedulerGlidein

        jobParams = ""

        (self.remoteHost,
         self.remoteUserHost) = self.pickRemoteSubmissionHost(task)

        psnDest = task.jobs[i - 1]['dlsDestination']

        if psnDest == ['']:
            # crab.cfg had datasetpath = None
            pnn2psn = getMapOfPhedexNodeName2ProcessingNodeNameFromSiteDB()
            allPSNs = set(pnn2psn.values())  # set removes duplicates
            psnDest = allPSNs

        blackList = parseIntoList(self.cfg_params.get("GRID.se_black_list",
                                                      []))
        whiteList = parseIntoList(self.cfg_params.get("GRID.se_white_list",
                                                      []))

        psnDest = cleanPsnListForBlackWhiteLists(psnDest, blackList, whiteList)
        if not psnDest or psnDest == [] or psnDest == ['']:
            msg = "No Processing Site Name after applying black/white list."
            msg += " Can't submit"
            msg += "\nUsed Black List: %s" % blackList
            msg += "\nUsed White List: %s" % whiteList
            common.logger.info(msg)
            raise CrabException(msg)

        # condor's JDL needs a string of CSV's, not a list
        psnDest = ",".join(psnDest)
        msg = "list of PSN's for submission: %s" % psnDest
        common.logger.info(msg)

        jobParams += '+DESIRED_Sites = "%s";' % psnDest

        scram = Scram.Scram(None)
        cmsVersion = scram.getSWVersion()
        scramArch = scram.getArch()

        cmsver = re.split('_', cmsVersion)
        if cmsver[3] == 'DEVEL': cmsver[3] = '0'
        numericCmsVersion = "%s%.2d%.2d" % (cmsver[1], int(
            cmsver[2]), int(cmsver[3]))

        if "slc5" in scramArch:
            cmsOpSys = "LINUX"
            cmsOpSysMajorVer = "5,6"  # SL5 exe's also run on SL6
        if "slc6" in scramArch:
            cmsOpSys = "LINUX"
            cmsOpSysMajorVer = "6"
        if "ia32" in scramArch:
            cmsArch = "INTEL,X86_64"  # 32bit exe's also run on 64bit
        if "amd64" in scramArch:
            cmsArch = "X86_64"

        # protect against datasetPath being None
        jobParams += '+DESIRED_CMSDataset ="' + str(self.datasetPath) + '";'

        jobParams += '+DESIRED_CMSVersion ="' + cmsVersion + '";'
        jobParams += '+DESIRED_CMSVersionNr ="' + numericCmsVersion + '";'
        jobParams += '+DESIRED_CMSScramArch ="' + scramArch + '";'
        jobParams += '+DESIRED_OpSyses ="' + cmsOpSys + '";'
        jobParams += '+DESIRED_OpSysMajorVers ="' + cmsOpSysMajorVer + '";'
        jobParams += '+DESIRED_Archs ="' + cmsArch + '";'

        userName = gethnUserNameFromSiteDB()
        jobParams += '+AccountingGroup = "analysis.%s";' % userName

        myscheddName = self.remoteHost

        jobParams += '+Glidein_MonitorID = "https://'+ myscheddName + \
                     '//' + self.submissionDay + '//$(Cluster).$(Process)"; '

        if (self.EDG_clock_time):
            glideinTime = "%d" % (int(self.EDG_clock_time) + 20
                                  )  # 20 min to wrapup
            jobParams += '+MaxWallTimeMins = ' + glideinTime + '; '
        else:
            jobParams += '+MaxWallTimeMins = %d; ' % (
                21 * 60 + 55)  #  21:55h  (unit = min)

        if self.max_rss:
            jobParams += 'request_memory = ' + self.max_rss + ';'

        if self.allowOverflow == "0":
            jobParams += '+CMS_ALLOW_OVERFLOW = False; '

        if self.EDG_addJdlParam:
            if self.EDG_addJdlParam[-1] == '':
                self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
            for p in self.EDG_addJdlParam:
                jobParams += p.strip() + ';\n'

        common._db.updateTask_({'jobType': jobParams})

        return jobParams