コード例 #1
0
    def __init__(self):
        SchedulerGrid.__init__(self,"RCONDOR")

        self.datasetPath   = None
        self.selectNoInput = None
        self.OSBsize = 50*1000*1000 # 50 MB

        self.environment_unique_identifier = None

        return
コード例 #2
0
    def __init__(self):
        SchedulerGrid.__init__(self, "RCONDOR")

        self.datasetPath = None
        self.selectNoInput = None
        self.OSBsize = 50 * 1000 * 1000  # 50 MB

        self.environment_unique_identifier = None

        return
コード例 #3
0
    def __init__(self):
        SchedulerGrid.__init__(self,"REMOTEGLIDEIN")

        self.datasetPath   = None
        self.selectNoInput = None
        self.OSBsize = 50*1000*1000 # 50 MB

        self.environment_unique_identifier = None
        self.submissionDay = time.strftime("%y%m%d",time.localtime())
        
        return
コード例 #4
0
    def __init__(self):
        SchedulerGrid.__init__(self, "REMOTEGLIDEIN")

        self.datasetPath = None
        self.selectNoInput = None
        self.OSBsize = 50 * 1000 * 1000  # 50 MB

        self.environment_unique_identifier = None
        self.submissionDay = time.strftime("%y%m%d", time.localtime())

        return
コード例 #5
0
    def __init__(self):
        SchedulerGrid.__init__(self,"RCONDOR")
        self.rcondorHost   = os.getenv('RCONDOR_HOST')
        if self.rcondorHost == None:
            raise CrabException('FATAL ERROR: env.var RCONDOR_HOST not defined')
        self.datasetPath   = None
        self.selectNoInput = None
        self.OSBsize = 50*1000*1000 # 50 MB

        self.environment_unique_identifier = None

        return
コード例 #6
0
ファイル: SchedulerArc.py プロジェクト: PerilousApricot/CRAB2
    def configure(self,cfg_params):

        if not os.environ.has_key('EDG_WL_LOCATION'):
            # This is an ugly hack needed for SchedulerGrid.configure() to
            # work!
            os.environ['EDG_WL_LOCATION'] = ''

        if not os.environ.has_key('X509_USER_PROXY'):
            # Set X509_USER_PROXY to the default location.  We'll do this
            # because in functions called by Scheduler.checkProxy()
            # voms-proxy-info will be called with '-file $X509_USER_PROXY',
            # so if X509_USER_PROXY isn't set, it won't work.
            os.environ['X509_USER_PROXY'] = '/tmp/x509up_u' + str(os.getuid())

        SchedulerGrid.configure(self, cfg_params)
        self.environment_unique_identifier = None
コード例 #7
0
    def configure(self, cfg_params):

        if not os.environ.has_key('EDG_WL_LOCATION'):
            # This is an ugly hack needed for SchedulerGrid.configure() to
            # work!
            os.environ['EDG_WL_LOCATION'] = ''

        if not os.environ.has_key('X509_USER_PROXY'):
            # Set X509_USER_PROXY to the default location.  We'll do this
            # because in functions called by Scheduler.checkProxy()
            # voms-proxy-info will be called with '-file $X509_USER_PROXY',
            # so if X509_USER_PROXY isn't set, it won't work.
            os.environ['X509_USER_PROXY'] = '/tmp/x509up_u' + str(os.getuid())

        SchedulerGrid.configure(self, cfg_params)
        self.environment_unique_identifier = None
コード例 #8
0
 def listMatch(self, dest, full):
     matching='fast'
     
     if self.boss().schedulerConfig['name'] == 'SchedulerGLite' :
         taskId=common._db.getTask()
         req=str(self.sched_parameter(1,taskId))
         sites = self.boss().schedSession().matchResources(taskId, requirements=req)
     else :
         sites = SchedulerGrid.listMatch(self, dest, full)
         
     if full == True: matching='full'
     common.logger.debug("list of available site ( "+str(matching) +" matching ) : "+str(sites))
     
     return sites
コード例 #9
0
    def configure(self, cfg_params):
        """
        Configure the scheduler with the config settings from the user
        """

        SchedulerGrid.configure(self, cfg_params)

        self.proxyValid=0
        self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0))
        self.space_token = cfg_params.get("USER.space_token",None)
        self.proxyServer= 'myproxy.cern.ch'
        self.group = cfg_params.get("GRID.group", None)
        self.role = cfg_params.get("GRID.role", None)
        self.VO = cfg_params.get('GRID.virtual_organization','cms')
        
        try:
            tmp =  cfg_params['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        if cfg_params.get('GRID.ce_black_list', None) or \
           cfg_params.get('GRID.ce_white_list', None) :
            msg="BEWARE: scheduler RGLIDEIN ignores CE black/white lists."
            msg+="\n Remove them from crab configuration to proceed."
            msg+="\n Use GRID.se_white_list and/or GRID.se_black_list instead"
            raise CrabException(msg)

        self.checkProxy()

        return
コード例 #10
0
    def listMatch(self, dest, full):
        matching = 'fast'

        if self.boss().schedulerConfig['name'] == 'SchedulerGLite':
            taskId = common._db.getTask()
            req = str(self.sched_parameter(1, taskId))
            sites = self.boss().schedSession().matchResources(taskId,
                                                              requirements=req)
        else:
            sites = SchedulerGrid.listMatch(self, dest, full)

        if full == True: matching = 'full'
        common.logger.debug("list of available site ( " + str(matching) +
                            " matching ) : " + str(sites))

        return sites
コード例 #11
0
 def __init__(self, name='ARC'):
     SchedulerGrid.__init__(self, name)
     self.OSBsize = None
     return
コード例 #12
0
 def configure(self,cfg_params):
     SchedulerGrid.configure(self, cfg_params)
     self.environment_unique_identifier = '$GLITE_WMS_JOBID'
コード例 #13
0
    def __init__(self, name="GLITE"):
        SchedulerGrid.__init__(self,name)

        self.EDG_retry_count        = 0
        self.EDG_shallow_retry_count= -1
        self.OSBsize = 55*1000*1000  # 55MB
コード例 #14
0
    def configure(self, cfgParams):
        """
        Configure the scheduler with the config settings from the user
        """
        # FIXME: Get rid of try/except and use get() instead

        if not os.environ.has_key('EDG_WL_LOCATION'):
            # This is an ugly hack needed for SchedulerGrid.configure() to
            # work!
            os.environ['EDG_WL_LOCATION'] = ''

        if not os.environ.has_key('X509_USER_PROXY'):
            # Set X509_USER_PROXY to the default location.  We'll do this
            # because in functions called by Scheduler.checkProxy()
            # voms-proxy-info will be called with '-file $X509_USER_PROXY',
            # so if X509_USER_PROXY isn't set, it won't work.
            os.environ['X509_USER_PROXY'] = '/tmp/x509up_u' + str(os.getuid())

        SchedulerGrid.configure(self, cfgParams)
        if cfgParams.get('CRAB.server_name',None) or cfgParams.get('CRAB.use_server',None):
            pass
        else:
            self.checkCondorSetup()

        # init BlackWhiteListParser
        ceWhiteList = cfgParams.get('GRID.ce_white_list',[])
        ceBlackList = cfgParams.get('GRID.ce_black_list',[])
        self.ceBlackWhiteListParser = \
            CEBlackWhiteListParser(ceWhiteList, ceBlackList, common.logger())

        try:
            self.GLOBUS_RSL = cfgParams['CONDORG.globus_rsl']
        except KeyError:
            self.GLOBUS_RSL = ''

        # Provide an override for the batchsystem that condor_g specifies
        # as a grid resource. This is to handle the case where the site
        # supports several batchsystem but bdii only allows a site
        # to public one.
        try:
            self.batchsystem = cfgParams['CONDORG.batchsystem']
            msg = self.msgPre + 'batchsystem overide specified in your crab.cfg'
            common.logger.debug(msg)
        except KeyError:
            self.batchsystem = ''

        self.datasetPath = ''

        try:
            tmp =  cfgParams['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        return
コード例 #15
0
    def configure(self, cfgParams):
        """
        Configure the scheduler with the config settings from the user
        """
        # FIXME: Get rid of try/except and use get() instead

        if not os.environ.has_key('EDG_WL_LOCATION'):
            # This is an ugly hack needed for SchedulerGrid.configure() to
            # work!
            os.environ['EDG_WL_LOCATION'] = ''

        if not os.environ.has_key('X509_USER_PROXY'):
            # Set X509_USER_PROXY to the default location.  We'll do this
            # because in functions called by Scheduler.checkProxy()
            # voms-proxy-info will be called with '-file $X509_USER_PROXY',
            # so if X509_USER_PROXY isn't set, it won't work.
            os.environ['X509_USER_PROXY'] = '/tmp/x509up_u' + str(os.getuid())

        SchedulerGrid.configure(self, cfgParams)
        if cfgParams.get('CRAB.server_name',None) or cfgParams.get('CRAB.use_server',None):
            pass
        else:
            self.checkCondorSetup()

        # init BlackWhiteListParser
        ceWhiteList = cfgParams.get('GRID.ce_white_list',[])
        ceBlackList = cfgParams.get('GRID.ce_black_list',[])
        self.ceBlackWhiteListParser = \
            CEBlackWhiteListParser(ceWhiteList, ceBlackList, common.logger())

        try:
            self.GLOBUS_RSL = cfgParams['CONDORG.globus_rsl']
        except KeyError:
            self.GLOBUS_RSL = ''

        # Provide an override for the batchsystem that condor_g specifies
        # as a grid resource. This is to handle the case where the site
        # supports several batchsystem but bdii only allows a site
        # to public one.
        try:
            self.batchsystem = cfgParams['CONDORG.batchsystem']
            msg = self.msgPre + 'batchsystem overide specified in your crab.cfg'
            common.logger.debug(msg)
        except KeyError:
            self.batchsystem = ''

        self.datasetPath = ''

        try:
            tmp =  cfgParams['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        return
コード例 #16
0
    def __init__(self, name="GLITE"):
        SchedulerGrid.__init__(self, name)

        self.EDG_retry_count = 0
        self.EDG_shallow_retry_count = -1
        self.OSBsize = 55 * 1000 * 1000  # 55MB
コード例 #17
0
    def configure(self, cfg_params):
        """
        Configure the scheduler with the config settings from the user
        """

        # this line needs to be before the call to SchedulerGrid.configure
        # because that calls SchedulerRemoteglidin in turn and
        # sshControlPersist needs to be defined then :-(
        self.sshControlPersist = cfg_params.get('USER.ssh_control_persist',
                                                '3600')
        if self.sshControlPersist.lower() == "no" or \
                self.sshControlPersist.lower() == "yes" or \
                self.sshControlPersist.isdigit() :
            pass
        else:
            msg = "Error: invalid value '%s' for USER.ssh_control_persist " % \
                self.sshControlPersist
            raise CrabException(msg)

        SchedulerGrid.configure(self, cfg_params)

        self.proxyValid = 0
        self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", '0'))
        self.space_token = cfg_params.get("USER.space_token", None)
        self.proxyServer = 'myproxy.cern.ch'
        self.group = cfg_params.get("GRID.group", None)
        self.role = cfg_params.get("GRID.role", None)
        self.VO = cfg_params.get('GRID.virtual_organization', 'cms')
        self.allowOverflow = cfg_params.get('GRID.allow_overflow', '1')
        self.max_rss = cfg_params.get('GRID.max_rss', '2000')

        self.checkProxy()

        try:
            tmp = cfg_params['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        if cfg_params.get('GRID.ce_black_list', None) or \
           cfg_params.get('GRID.ce_white_list', None) :
            msg = "BEWARE: scheduler REMOTEGLIDEIN ignores CE black/white lists."
            msg += "\n Remove them from crab configuration to proceed."
            msg += "\n Use GRID.se_white_list and/or GRID.se_black_list instead"
            raise CrabException(msg)

        # make sure proxy FQAN has not changed since last time
        command = "voms-proxy-info -identity -fqan 2>/dev/null"
        command += " | head -2"
        identity = runCommand(command)
        idfile = common.work_space.shareDir() + "GridIdentity"
        if os.access(idfile, os.F_OK):
            # identity file exists from previous commands
            f = open(idfile, 'r')
            idFromFile = f.read()
            f.close()
        else:
            # create it
            f = open(idfile, 'w')
            f.write(identity)
            f.close()
            idFromFile = identity

        if identity != idFromFile:
            msg = "Wrong Grid Credentials:\n%s" % identity
            msg += "\nMake sure you have "
            msg += " DN, FQAN =\n%s" % idFromFile
            raise CrabException(msg)

        return
コード例 #18
0
    def configure(self, cfg_params):
        """
        Configure the scheduler with the config settings from the user
        """
        
#        task  = common._db.getTask()
#        #print task.__dict__
#
#        if task['serverName']!=None and task['serverName']!="":
#            # cast to string to avoid issues with unicode :-(
#            self.rcondorUserHost=str(task['serverName'])
#            common.logger.info("serverName from Task DB is %s" %
#                               self.rcondorUserHost)
#        else :
#            # get an rcondor host from config and save
#            common.logger.info("no serverName in Task DB, use env.var.")
#
#            self.rcondorHost   = os.getenv('RCONDOR_HOST')
#            if not self.rcondorHost :
#                raise CrabException('FATAL ERROR: env.var RCONDOR_HOST not defined')
#            self.rcondorUser = os.getenv('RCONDOR_USER')
#            if not self.rcondorUser :
#                common.logger.info("$RCONDOR_USER not defined, try to find out via uberftp ...")
#                command="uberftp $RCONDOR_HOST pwd|grep User|awk '{print $3}'"
#                (status, output) = commands.getstatusoutput(command)
#                if status == 0:
#                    self.rcondorUser = output
#                    common.logger.info("rcondorUser set to %s" % self.rcondorUser)
#                if self.rcondorUser==None:
#                    raise CrabException('FATAL ERROR: RCONDOR_USER not defined')
#
#                self.rcondorUserHost = self.rcondorUser + '@' + self.rcondorHost
#
#            print "will set server name to : ", self.rcondorUserHost
#            common._db.updateTask_({'serverName':self.rcondorUserHost})
#            print "ok"

        SchedulerGrid.configure(self, cfg_params)

        self.proxyValid=0
        self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0))
        self.space_token = cfg_params.get("USER.space_token",None)
        self.proxyServer= 'myproxy.cern.ch'
        self.group = cfg_params.get("GRID.group", None)
        self.role = cfg_params.get("GRID.role", None)
        self.VO = cfg_params.get('GRID.virtual_organization','cms')

        self.checkProxy()


        try:
            tmp =  cfg_params['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        if cfg_params.get('GRID.ce_black_list', None) or \
           cfg_params.get('GRID.ce_white_list', None) :
            msg="BEWARE: scheduler RGLIDEIN ignores CE black/white lists."
            msg+="\n Remove them from crab configuration to proceed."
            msg+="\n Use GRID.se_white_list and/or GRID.se_black_list instead"
            raise CrabException(msg)

        return
コード例 #19
0
    def __init__(self, name):
        SchedulerGrid.__init__(self, name)
        self.environment_unique_identifier = None
        self.msgPre = '[Condor-G Scheduler]: '

        return
コード例 #20
0
    def configure(self, cfg_params):
        """
        Configure the scheduler with the config settings from the user
        """
   
        # this line needs to be before the call to SchedulerGrid.configure
        # because that calls SchedulerRemoteglidin in turn and
        # sshControlPersist needs to be defined then :-(
        self.sshControlPersist =  cfg_params.get('USER.ssh_control_persist','3600')
        if self.sshControlPersist.lower() == "no" or \
                self.sshControlPersist.lower() == "yes" or \
                self.sshControlPersist.isdigit() :
            pass
        else:
            msg = "Error: invalid value '%s' for USER.ssh_control_persist " % \
                self.sshControlPersist
            raise CrabException(msg)

        SchedulerGrid.configure(self, cfg_params)

        self.proxyValid=0
        self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",'0'))
        self.space_token = cfg_params.get("USER.space_token",None)
        self.proxyServer= 'myproxy.cern.ch'
        self.group = cfg_params.get("GRID.group", None)
        self.role = cfg_params.get("GRID.role", None)
        self.VO = cfg_params.get('GRID.virtual_organization','cms')
        self.allowOverflow = cfg_params.get('GRID.allow_overflow', '1')
        self.max_rss = cfg_params.get('GRID.max_rss','2000')

        self.checkProxy()


        try:
            tmp =  cfg_params['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        if cfg_params.get('GRID.ce_black_list', None) or \
           cfg_params.get('GRID.ce_white_list', None) :
            msg="BEWARE: scheduler REMOTEGLIDEIN ignores CE black/white lists."
            msg+="\n Remove them from crab configuration to proceed."
            msg+="\n Use GRID.se_white_list and/or GRID.se_black_list instead"
            raise CrabException(msg)


        # make sure proxy FQAN has not changed since last time
        command =  "eval `scram unsetenv -sh`; voms-proxy-info -identity -fqan 2>/dev/null"
        command += " | head -2"
        identity = runCommand(command)
        idfile = common.work_space.shareDir() + "GridIdentity"
        if os.access(idfile, os.F_OK) :
            # identity file exists from previous commands
            f=open(idfile, 'r')
            idFromFile=f.read()
            f.close()
        else :
            # create it
            f=open(idfile, 'w')
            f.write(identity)
            f.close()
            idFromFile = identity

        if identity != idFromFile:
            msg =  "Wrong Grid Credentials:\n%s" % identity
            msg += "\nMake sure you have "
            msg += " DN, FQAN =\n%s" % idFromFile
            raise CrabException(msg)

        return
コード例 #21
0
 def configure(self, cfg_params):
     SchedulerGrid.configure(self, cfg_params)
     self.environment_unique_identifier = '$GLITE_WMS_JOBID'
コード例 #22
0
    def configure(self, cfg_params):
        """
        Configure the scheduler with the config settings from the user
        """

        #        task  = common._db.getTask()
        #        #print task.__dict__
        #
        #        if task['serverName']!=None and task['serverName']!="":
        #            # cast to string to avoid issues with unicode :-(
        #            self.rcondorUserHost=str(task['serverName'])
        #            common.logger.info("serverName from Task DB is %s" %
        #                               self.rcondorUserHost)
        #        else :
        #            # get an rcondor host from config and save
        #            common.logger.info("no serverName in Task DB, use env.var.")
        #
        #            self.rcondorHost   = os.getenv('RCONDOR_HOST')
        #            if not self.rcondorHost :
        #                raise CrabException('FATAL ERROR: env.var RCONDOR_HOST not defined')
        #            self.rcondorUser = os.getenv('RCONDOR_USER')
        #            if not self.rcondorUser :
        #                common.logger.info("$RCONDOR_USER not defined, try to find out via uberftp ...")
        #                command="uberftp $RCONDOR_HOST pwd|grep User|awk '{print $3}'"
        #                (status, output) = commands.getstatusoutput(command)
        #                if status == 0:
        #                    self.rcondorUser = output
        #                    common.logger.info("rcondorUser set to %s" % self.rcondorUser)
        #                if self.rcondorUser==None:
        #                    raise CrabException('FATAL ERROR: RCONDOR_USER not defined')
        #
        #                self.rcondorUserHost = self.rcondorUser + '@' + self.rcondorHost
        #
        #            print "will set server name to : ", self.rcondorUserHost
        #            common._db.updateTask_({'serverName':self.rcondorUserHost})
        #            print "ok"

        SchedulerGrid.configure(self, cfg_params)

        self.proxyValid = 0
        self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", 0))
        self.space_token = cfg_params.get("USER.space_token", None)
        self.proxyServer = 'myproxy.cern.ch'
        self.group = cfg_params.get("GRID.group", None)
        self.role = cfg_params.get("GRID.role", None)
        self.VO = cfg_params.get('GRID.virtual_organization', 'cms')

        self.checkProxy()

        try:
            tmp = cfg_params['CMSSW.datasetpath']
            if tmp.lower() == 'none':
                self.datasetPath = None
                self.selectNoInput = 1
            else:
                self.datasetPath = tmp
                self.selectNoInput = 0
        except KeyError:
            msg = "Error: datasetpath not defined "
            raise CrabException(msg)

        if cfg_params.get('GRID.ce_black_list', None) or \
           cfg_params.get('GRID.ce_white_list', None) :
            msg = "BEWARE: scheduler RGLIDEIN ignores CE black/white lists."
            msg += "\n Remove them from crab configuration to proceed."
            msg += "\n Use GRID.se_white_list and/or GRID.se_black_list instead"
            raise CrabException(msg)

        return
コード例 #23
0
ファイル: SchedulerArc.py プロジェクト: jpata/CRAB2
 def __init__(self, name='ARC'):
     SchedulerGrid.__init__(self,name)
     self.OSBsize = None
     return
コード例 #24
0
    def __init__(self, name):
        SchedulerGrid.__init__(self, name)
        self.environment_unique_identifier = None
        self.msgPre = '[Condor-G Scheduler]: '

        return