def __init__(self):
SchedulerGrid.__init__(self,"RCONDOR")
self.datasetPath = None
self.selectNoInput = None
self.OSBsize = 50*1000*1000 # 50 MB
self.environment_unique_identifier = None
return
def __init__(self):
SchedulerGrid.__init__(self, "RCONDOR")
self.datasetPath = None
self.selectNoInput = None
self.OSBsize = 50 * 1000 * 1000 # 50 MB
self.environment_unique_identifier = None
return
def __init__(self):
SchedulerGrid.__init__(self,"REMOTEGLIDEIN")
self.datasetPath = None
self.selectNoInput = None
self.OSBsize = 50*1000*1000 # 50 MB
self.environment_unique_identifier = None
self.submissionDay = time.strftime("%y%m%d",time.localtime())
return
def __init__(self):
SchedulerGrid.__init__(self, "REMOTEGLIDEIN")
self.datasetPath = None
self.selectNoInput = None
self.OSBsize = 50 * 1000 * 1000 # 50 MB
self.environment_unique_identifier = None
self.submissionDay = time.strftime("%y%m%d", time.localtime())
return
def __init__(self):
SchedulerGrid.__init__(self,"RCONDOR")
self.rcondorHost = os.getenv('RCONDOR_HOST')
if self.rcondorHost == None:
raise CrabException('FATAL ERROR: env.var RCONDOR_HOST not defined')
self.datasetPath = None
self.selectNoInput = None
self.OSBsize = 50*1000*1000 # 50 MB
self.environment_unique_identifier = None
return
def configure(self,cfg_params):
if not os.environ.has_key('EDG_WL_LOCATION'):
# This is an ugly hack needed for SchedulerGrid.configure() to
# work!
os.environ['EDG_WL_LOCATION'] = ''
if not os.environ.has_key('X509_USER_PROXY'):
# Set X509_USER_PROXY to the default location. We'll do this
# because in functions called by Scheduler.checkProxy()
# voms-proxy-info will be called with '-file $X509_USER_PROXY',
# so if X509_USER_PROXY isn't set, it won't work.
os.environ['X509_USER_PROXY'] = '/tmp/x509up_u' + str(os.getuid())
SchedulerGrid.configure(self, cfg_params)
self.environment_unique_identifier = None
def configure(self, cfg_params):
if not os.environ.has_key('EDG_WL_LOCATION'):
# This is an ugly hack needed for SchedulerGrid.configure() to
# work!
os.environ['EDG_WL_LOCATION'] = ''
if not os.environ.has_key('X509_USER_PROXY'):
# Set X509_USER_PROXY to the default location. We'll do this
# because in functions called by Scheduler.checkProxy()
# voms-proxy-info will be called with '-file $X509_USER_PROXY',
# so if X509_USER_PROXY isn't set, it won't work.
os.environ['X509_USER_PROXY'] = '/tmp/x509up_u' + str(os.getuid())
SchedulerGrid.configure(self, cfg_params)
self.environment_unique_identifier = None
def listMatch(self, dest, full):
matching='fast'
if self.boss().schedulerConfig['name'] == 'SchedulerGLite' :
taskId=common._db.getTask()
req=str(self.sched_parameter(1,taskId))
sites = self.boss().schedSession().matchResources(taskId, requirements=req)
else :
sites = SchedulerGrid.listMatch(self, dest, full)
if full == True: matching='full'
common.logger.debug("list of available site ( "+str(matching) +" matching ) : "+str(sites))
return sites
def configure(self, cfg_params):
"""
Configure the scheduler with the config settings from the user
"""
SchedulerGrid.configure(self, cfg_params)
self.proxyValid=0
self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0))
self.space_token = cfg_params.get("USER.space_token",None)
self.proxyServer= 'myproxy.cern.ch'
self.group = cfg_params.get("GRID.group", None)
self.role = cfg_params.get("GRID.role", None)
self.VO = cfg_params.get('GRID.virtual_organization','cms')
try:
tmp = cfg_params['CMSSW.datasetpath']
if tmp.lower() == 'none':
self.datasetPath = None
self.selectNoInput = 1
else:
self.datasetPath = tmp
self.selectNoInput = 0
except KeyError:
msg = "Error: datasetpath not defined "
raise CrabException(msg)
if cfg_params.get('GRID.ce_black_list', None) or \
cfg_params.get('GRID.ce_white_list', None) :
msg="BEWARE: scheduler RGLIDEIN ignores CE black/white lists."
msg+="\n Remove them from crab configuration to proceed."
msg+="\n Use GRID.se_white_list and/or GRID.se_black_list instead"
raise CrabException(msg)
self.checkProxy()
return
def listMatch(self, dest, full):
matching = 'fast'
if self.boss().schedulerConfig['name'] == 'SchedulerGLite':
taskId = common._db.getTask()
req = str(self.sched_parameter(1, taskId))
sites = self.boss().schedSession().matchResources(taskId,
requirements=req)
else:
sites = SchedulerGrid.listMatch(self, dest, full)
if full == True: matching = 'full'
common.logger.debug("list of available site ( " + str(matching) +
" matching ) : " + str(sites))
return sites
def __init__(self, name='ARC'):
SchedulerGrid.__init__(self, name)
self.OSBsize = None
return
def configure(self,cfg_params):
SchedulerGrid.configure(self, cfg_params)
self.environment_unique_identifier = '$GLITE_WMS_JOBID'
def __init__(self, name="GLITE"):
SchedulerGrid.__init__(self,name)
self.EDG_retry_count = 0
self.EDG_shallow_retry_count= -1
self.OSBsize = 55*1000*1000 # 55MB
def configure(self, cfgParams):
"""
Configure the scheduler with the config settings from the user
"""
# FIXME: Get rid of try/except and use get() instead
if not os.environ.has_key('EDG_WL_LOCATION'):
# This is an ugly hack needed for SchedulerGrid.configure() to
# work!
os.environ['EDG_WL_LOCATION'] = ''
if not os.environ.has_key('X509_USER_PROXY'):
# Set X509_USER_PROXY to the default location. We'll do this
# because in functions called by Scheduler.checkProxy()
# voms-proxy-info will be called with '-file $X509_USER_PROXY',
# so if X509_USER_PROXY isn't set, it won't work.
os.environ['X509_USER_PROXY'] = '/tmp/x509up_u' + str(os.getuid())
SchedulerGrid.configure(self, cfgParams)
if cfgParams.get('CRAB.server_name',None) or cfgParams.get('CRAB.use_server',None):
pass
else:
self.checkCondorSetup()
# init BlackWhiteListParser
ceWhiteList = cfgParams.get('GRID.ce_white_list',[])
ceBlackList = cfgParams.get('GRID.ce_black_list',[])
self.ceBlackWhiteListParser = \
CEBlackWhiteListParser(ceWhiteList, ceBlackList, common.logger())
try:
self.GLOBUS_RSL = cfgParams['CONDORG.globus_rsl']
except KeyError:
self.GLOBUS_RSL = ''
# Provide an override for the batchsystem that condor_g specifies
# as a grid resource. This is to handle the case where the site
# supports several batchsystem but bdii only allows a site
# to public one.
try:
self.batchsystem = cfgParams['CONDORG.batchsystem']
msg = self.msgPre + 'batchsystem overide specified in your crab.cfg'
common.logger.debug(msg)
except KeyError:
self.batchsystem = ''
self.datasetPath = ''
try:
tmp = cfgParams['CMSSW.datasetpath']
if tmp.lower() == 'none':
self.datasetPath = None
self.selectNoInput = 1
else:
self.datasetPath = tmp
self.selectNoInput = 0
except KeyError:
msg = "Error: datasetpath not defined "
raise CrabException(msg)
return
def configure(self, cfgParams):
"""
Configure the scheduler with the config settings from the user
"""
# FIXME: Get rid of try/except and use get() instead
if not os.environ.has_key('EDG_WL_LOCATION'):
# This is an ugly hack needed for SchedulerGrid.configure() to
# work!
os.environ['EDG_WL_LOCATION'] = ''
if not os.environ.has_key('X509_USER_PROXY'):
# Set X509_USER_PROXY to the default location. We'll do this
# because in functions called by Scheduler.checkProxy()
# voms-proxy-info will be called with '-file $X509_USER_PROXY',
# so if X509_USER_PROXY isn't set, it won't work.
os.environ['X509_USER_PROXY'] = '/tmp/x509up_u' + str(os.getuid())
SchedulerGrid.configure(self, cfgParams)
if cfgParams.get('CRAB.server_name',None) or cfgParams.get('CRAB.use_server',None):
pass
else:
self.checkCondorSetup()
# init BlackWhiteListParser
ceWhiteList = cfgParams.get('GRID.ce_white_list',[])
ceBlackList = cfgParams.get('GRID.ce_black_list',[])
self.ceBlackWhiteListParser = \
CEBlackWhiteListParser(ceWhiteList, ceBlackList, common.logger())
try:
self.GLOBUS_RSL = cfgParams['CONDORG.globus_rsl']
except KeyError:
self.GLOBUS_RSL = ''
# Provide an override for the batchsystem that condor_g specifies
# as a grid resource. This is to handle the case where the site
# supports several batchsystem but bdii only allows a site
# to public one.
try:
self.batchsystem = cfgParams['CONDORG.batchsystem']
msg = self.msgPre + 'batchsystem overide specified in your crab.cfg'
common.logger.debug(msg)
except KeyError:
self.batchsystem = ''
self.datasetPath = ''
try:
tmp = cfgParams['CMSSW.datasetpath']
if tmp.lower() == 'none':
self.datasetPath = None
self.selectNoInput = 1
else:
self.datasetPath = tmp
self.selectNoInput = 0
except KeyError:
msg = "Error: datasetpath not defined "
raise CrabException(msg)
return
def __init__(self, name="GLITE"):
SchedulerGrid.__init__(self, name)
self.EDG_retry_count = 0
self.EDG_shallow_retry_count = -1
self.OSBsize = 55 * 1000 * 1000 # 55MB
def configure(self, cfg_params):
"""
Configure the scheduler with the config settings from the user
"""
# this line needs to be before the call to SchedulerGrid.configure
# because that calls SchedulerRemoteglidin in turn and
# sshControlPersist needs to be defined then :-(
self.sshControlPersist = cfg_params.get('USER.ssh_control_persist',
'3600')
if self.sshControlPersist.lower() == "no" or \
self.sshControlPersist.lower() == "yes" or \
self.sshControlPersist.isdigit() :
pass
else:
msg = "Error: invalid value '%s' for USER.ssh_control_persist " % \
self.sshControlPersist
raise CrabException(msg)
SchedulerGrid.configure(self, cfg_params)
self.proxyValid = 0
self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", '0'))
self.space_token = cfg_params.get("USER.space_token", None)
self.proxyServer = 'myproxy.cern.ch'
self.group = cfg_params.get("GRID.group", None)
self.role = cfg_params.get("GRID.role", None)
self.VO = cfg_params.get('GRID.virtual_organization', 'cms')
self.allowOverflow = cfg_params.get('GRID.allow_overflow', '1')
self.max_rss = cfg_params.get('GRID.max_rss', '2000')
self.checkProxy()
try:
tmp = cfg_params['CMSSW.datasetpath']
if tmp.lower() == 'none':
self.datasetPath = None
self.selectNoInput = 1
else:
self.datasetPath = tmp
self.selectNoInput = 0
except KeyError:
msg = "Error: datasetpath not defined "
raise CrabException(msg)
if cfg_params.get('GRID.ce_black_list', None) or \
cfg_params.get('GRID.ce_white_list', None) :
msg = "BEWARE: scheduler REMOTEGLIDEIN ignores CE black/white lists."
msg += "\n Remove them from crab configuration to proceed."
msg += "\n Use GRID.se_white_list and/or GRID.se_black_list instead"
raise CrabException(msg)
# make sure proxy FQAN has not changed since last time
command = "voms-proxy-info -identity -fqan 2>/dev/null"
command += " | head -2"
identity = runCommand(command)
idfile = common.work_space.shareDir() + "GridIdentity"
if os.access(idfile, os.F_OK):
# identity file exists from previous commands
f = open(idfile, 'r')
idFromFile = f.read()
f.close()
else:
# create it
f = open(idfile, 'w')
f.write(identity)
f.close()
idFromFile = identity
if identity != idFromFile:
msg = "Wrong Grid Credentials:\n%s" % identity
msg += "\nMake sure you have "
msg += " DN, FQAN =\n%s" % idFromFile
raise CrabException(msg)
return
def configure(self, cfg_params):
"""
Configure the scheduler with the config settings from the user
"""
# task = common._db.getTask()
# #print task.__dict__
#
# if task['serverName']!=None and task['serverName']!="":
# # cast to string to avoid issues with unicode :-(
# self.rcondorUserHost=str(task['serverName'])
# common.logger.info("serverName from Task DB is %s" %
# self.rcondorUserHost)
# else :
# # get an rcondor host from config and save
# common.logger.info("no serverName in Task DB, use env.var.")
#
# self.rcondorHost = os.getenv('RCONDOR_HOST')
# if not self.rcondorHost :
# raise CrabException('FATAL ERROR: env.var RCONDOR_HOST not defined')
# self.rcondorUser = os.getenv('RCONDOR_USER')
# if not self.rcondorUser :
# common.logger.info("$RCONDOR_USER not defined, try to find out via uberftp ...")
# command="uberftp $RCONDOR_HOST pwd|grep User|awk '{print $3}'"
# (status, output) = commands.getstatusoutput(command)
# if status == 0:
# self.rcondorUser = output
# common.logger.info("rcondorUser set to %s" % self.rcondorUser)
# if self.rcondorUser==None:
# raise CrabException('FATAL ERROR: RCONDOR_USER not defined')
#
# self.rcondorUserHost = self.rcondorUser + '@' + self.rcondorHost
#
# print "will set server name to : ", self.rcondorUserHost
# common._db.updateTask_({'serverName':self.rcondorUserHost})
# print "ok"
SchedulerGrid.configure(self, cfg_params)
self.proxyValid=0
self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0))
self.space_token = cfg_params.get("USER.space_token",None)
self.proxyServer= 'myproxy.cern.ch'
self.group = cfg_params.get("GRID.group", None)
self.role = cfg_params.get("GRID.role", None)
self.VO = cfg_params.get('GRID.virtual_organization','cms')
self.checkProxy()
try:
tmp = cfg_params['CMSSW.datasetpath']
if tmp.lower() == 'none':
self.datasetPath = None
self.selectNoInput = 1
else:
self.datasetPath = tmp
self.selectNoInput = 0
except KeyError:
msg = "Error: datasetpath not defined "
raise CrabException(msg)
if cfg_params.get('GRID.ce_black_list', None) or \
cfg_params.get('GRID.ce_white_list', None) :
msg="BEWARE: scheduler RGLIDEIN ignores CE black/white lists."
msg+="\n Remove them from crab configuration to proceed."
msg+="\n Use GRID.se_white_list and/or GRID.se_black_list instead"
raise CrabException(msg)
return
def __init__(self, name):
SchedulerGrid.__init__(self, name)
self.environment_unique_identifier = None
self.msgPre = '[Condor-G Scheduler]: '
return
def configure(self, cfg_params):
"""
Configure the scheduler with the config settings from the user
"""
# this line needs to be before the call to SchedulerGrid.configure
# because that calls SchedulerRemoteglidin in turn and
# sshControlPersist needs to be defined then :-(
self.sshControlPersist = cfg_params.get('USER.ssh_control_persist','3600')
if self.sshControlPersist.lower() == "no" or \
self.sshControlPersist.lower() == "yes" or \
self.sshControlPersist.isdigit() :
pass
else:
msg = "Error: invalid value '%s' for USER.ssh_control_persist " % \
self.sshControlPersist
raise CrabException(msg)
SchedulerGrid.configure(self, cfg_params)
self.proxyValid=0
self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",'0'))
self.space_token = cfg_params.get("USER.space_token",None)
self.proxyServer= 'myproxy.cern.ch'
self.group = cfg_params.get("GRID.group", None)
self.role = cfg_params.get("GRID.role", None)
self.VO = cfg_params.get('GRID.virtual_organization','cms')
self.allowOverflow = cfg_params.get('GRID.allow_overflow', '1')
self.max_rss = cfg_params.get('GRID.max_rss','2000')
self.checkProxy()
try:
tmp = cfg_params['CMSSW.datasetpath']
if tmp.lower() == 'none':
self.datasetPath = None
self.selectNoInput = 1
else:
self.datasetPath = tmp
self.selectNoInput = 0
except KeyError:
msg = "Error: datasetpath not defined "
raise CrabException(msg)
if cfg_params.get('GRID.ce_black_list', None) or \
cfg_params.get('GRID.ce_white_list', None) :
msg="BEWARE: scheduler REMOTEGLIDEIN ignores CE black/white lists."
msg+="\n Remove them from crab configuration to proceed."
msg+="\n Use GRID.se_white_list and/or GRID.se_black_list instead"
raise CrabException(msg)
# make sure proxy FQAN has not changed since last time
command = "eval `scram unsetenv -sh`; voms-proxy-info -identity -fqan 2>/dev/null"
command += " | head -2"
identity = runCommand(command)
idfile = common.work_space.shareDir() + "GridIdentity"
if os.access(idfile, os.F_OK) :
# identity file exists from previous commands
f=open(idfile, 'r')
idFromFile=f.read()
f.close()
else :
# create it
f=open(idfile, 'w')
f.write(identity)
f.close()
idFromFile = identity
if identity != idFromFile:
msg = "Wrong Grid Credentials:\n%s" % identity
msg += "\nMake sure you have "
msg += " DN, FQAN =\n%s" % idFromFile
raise CrabException(msg)
return
def configure(self, cfg_params):
SchedulerGrid.configure(self, cfg_params)
self.environment_unique_identifier = '$GLITE_WMS_JOBID'
def configure(self, cfg_params):
"""
Configure the scheduler with the config settings from the user
"""
# task = common._db.getTask()
# #print task.__dict__
#
# if task['serverName']!=None and task['serverName']!="":
# # cast to string to avoid issues with unicode :-(
# self.rcondorUserHost=str(task['serverName'])
# common.logger.info("serverName from Task DB is %s" %
# self.rcondorUserHost)
# else :
# # get an rcondor host from config and save
# common.logger.info("no serverName in Task DB, use env.var.")
#
# self.rcondorHost = os.getenv('RCONDOR_HOST')
# if not self.rcondorHost :
# raise CrabException('FATAL ERROR: env.var RCONDOR_HOST not defined')
# self.rcondorUser = os.getenv('RCONDOR_USER')
# if not self.rcondorUser :
# common.logger.info("$RCONDOR_USER not defined, try to find out via uberftp ...")
# command="uberftp $RCONDOR_HOST pwd|grep User|awk '{print $3}'"
# (status, output) = commands.getstatusoutput(command)
# if status == 0:
# self.rcondorUser = output
# common.logger.info("rcondorUser set to %s" % self.rcondorUser)
# if self.rcondorUser==None:
# raise CrabException('FATAL ERROR: RCONDOR_USER not defined')
#
# self.rcondorUserHost = self.rcondorUser + '@' + self.rcondorHost
#
# print "will set server name to : ", self.rcondorUserHost
# common._db.updateTask_({'serverName':self.rcondorUserHost})
# print "ok"
SchedulerGrid.configure(self, cfg_params)
self.proxyValid = 0
self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", 0))
self.space_token = cfg_params.get("USER.space_token", None)
self.proxyServer = 'myproxy.cern.ch'
self.group = cfg_params.get("GRID.group", None)
self.role = cfg_params.get("GRID.role", None)
self.VO = cfg_params.get('GRID.virtual_organization', 'cms')
self.checkProxy()
try:
tmp = cfg_params['CMSSW.datasetpath']
if tmp.lower() == 'none':
self.datasetPath = None
self.selectNoInput = 1
else:
self.datasetPath = tmp
self.selectNoInput = 0
except KeyError:
msg = "Error: datasetpath not defined "
raise CrabException(msg)
if cfg_params.get('GRID.ce_black_list', None) or \
cfg_params.get('GRID.ce_white_list', None) :
msg = "BEWARE: scheduler RGLIDEIN ignores CE black/white lists."
msg += "\n Remove them from crab configuration to proceed."
msg += "\n Use GRID.se_white_list and/or GRID.se_black_list instead"
raise CrabException(msg)
return
def __init__(self, name='ARC'):
SchedulerGrid.__init__(self,name)
self.OSBsize = None
return
def __init__(self, name):
SchedulerGrid.__init__(self, name)
self.environment_unique_identifier = None
self.msgPre = '[Condor-G Scheduler]: '
return
