def __init__(self, cfg_params, nj_list):
PostMortem.__init__(self, cfg_params, nj_list)
# init client server params...
CliServerParams(self)
self.copyTout = setLcgTimeout()
if common.scheduler.name().upper() in ['LSF', 'CAF']:
self.copyTout = ' '
if self.storage_path[0] != '/':
self.storage_path = '/' + self.storage_path
return
def __init__(self, cfg_params, nj_list):
PostMortem.__init__(self, cfg_params, nj_list)
# init client server params...
CliServerParams(self)
self.copyTout= setLcgTimeout()
if common.scheduler.name().upper() in ['LSF', 'CAF']:
self.copyTout= ' '
if self.storage_path[0]!='/':
self.storage_path = '/'+self.storage_path
return
## get the list of jobs to get logging.info skimmed by failed status
logginable = self.skimDeadList()
if self.storage_proto in ['globus']:
for id in self.nj_list:
if id not in self.all_jobs:
common.logger.info(
'Warning: job # ' + str(id) +
' does not exist! Not possible to ask for postMortem ')
elif id not in logginable:
common.logger.info(
'Warning: job # ' + str(id) +
' not killed or aborted! Will get loggingInfo manually '
)
PostMortem.collectOneLogging(self, id)
# construct a list of absolute paths of input files
# and the destinations to copy them to
sourcesList = []
destsList = []
self.taskuuid = str(common._db.queryTask('name'))
common.logger.debug("Starting globus retrieval for task name: " +
self.taskuuid)
remotedir = os.path.join(self.storage_path, self.taskuuid)
for i in logginable:
remotelog = remotedir + '/loggingInfo_' + str(i) + '.log'
sourcesList.append(remotelog)
fname = self.fname_base + str(i) + '.LoggingInfo'
destsList.append(fname)
# try to do the copy
msg = "ERROR: Unable to create destination interface \n"
raise CrabException(msg)
## coupling se interfaces
sbi = SBinterface(seEl, loc, logger = common.logger.logger)
## get the list of jobs to get logging.info skimmed by failed status
logginable = self.skimDeadList()
if self.storage_proto in ['globus']:
for id in self.nj_list:
if id not in self.all_jobs:
common.logger.info('Warning: job # ' + str(id) + ' does not exist! Not possible to ask for postMortem ')
elif id not in logginable:
common.logger.info('Warning: job # ' + str(id) + ' not killed or aborted! Will get loggingInfo manually ')
PostMortem.collectOneLogging(self,id)
# construct a list of absolute paths of input files
# and the destinations to copy them to
sourcesList = []
destsList = []
self.taskuuid = str(common._db.queryTask('name'))
common.logger.debug( "Starting globus retrieval for task name: " + self.taskuuid)
remotedir = os.path.join(self.storage_path, self.taskuuid)
for i in logginable:
remotelog = remotedir + '/loggingInfo_'+str(i)+'.log'
sourcesList.append(remotelog)
fname = self.fname_base + str(i) + '.LoggingInfo'
destsList.append(fname)
# try to do the copy
copy_res = None
def initializeActions_(self, opts):
"""
For each user action instantiate a corresponding
object and put it in the action dictionary.
"""
for opt in opts.keys():
val = opts[opt]
if ( opt == '-create' ):
if self.flag_continue:
msg = 'Cannot create an existing project. \n'
raise CrabException(msg)
if val and val != 'all':
msg = 'Per default, CRAB will create all jobs as specified in the crab.cfg file, not the command line!'
common.logger.info(msg)
msg = 'Submission will still take into account the number of jobs specified on the command line!\n'
common.logger.info(msg)
ncjobs = 'all'
#wmbs
if int(self.automation) == 1:
msg = 'Your jobs will be created on the server. Jobs informations will be known later.'
common.logger.info(msg)
# wmbs
from Requestor import Requestor
# Instantiate Creator object
self.creator = Requestor(self.job_type_name,
self.cfg_params,
ncjobs)
self.actions[opt] = self.creator
# Create and initialize JobList
common.job_list = JobList(1,self.creator.jobType())
else:
from Creator import Creator
# Instantiate Creator object
self.creator = Creator(self.job_type_name,
self.cfg_params,
ncjobs)
self.actions[opt] = self.creator
# create jobs in the DB
common._db.createJobs_(self.creator.nJobsL())
# Create and initialize JobList
common.job_list = JobList(common._db.nJobs(),
self.creator.jobType())
## jobs specs not needed if using WMBS
self.creator.writeJobsSpecsToDB()
taskinfo={}
taskinfo['cfgName'] = common.work_space.jobDir()+"/"+self.creator.jobType().configFilename()
taskinfo['dataset'] = self.cfg_params['CMSSW.datasetpath']
common.job_list.setScriptNames(self.job_type_name+'.sh')
common.job_list.setCfgNames(self.creator.jobType().configFilename())
common._db.updateTask_(taskinfo)
pass
elif ( opt == '-submit' ):
## Dealt with val == int so that -submit N means submit N jobs and not job # N
if (self.UseServer== 1):
from SubmitterServer import SubmitterServer
self.actions[opt] = SubmitterServer(self.cfg_params, self.parseRange_(val), val)
else:
from Submitter import Submitter
# Instantiate Submitter object
self.actions[opt] = Submitter(self.cfg_params, self.parseRange_(val), val)
# Create and initialize JobList
if len(common.job_list) == 0 :
common.job_list = JobList(common._db.nJobs(),
None)
pass
pass
elif ( opt == '-list' ):
'''
Print the relevant infos of a range-all jobs/task
'''
jobs = self.parseRange_(val)
common._db.dump(jobs)
pass
elif ( opt == '-printId' ):
'''
Print the unique name of the task if crab is used as client
Print the SID list of all the jobs
'''
#jid=False
#if val == 'full': jid=True
jid=True
common._db.queryID(self.UseServer,jid)
elif ( opt == '-status' ):
if (self.UseServer== 1):
from StatusServer import StatusServer
self.actions[opt] = StatusServer(self.cfg_params, val)
else:
from Status import Status
self.actions[opt] = Status(self.cfg_params, val)
elif ( opt == '-kill' ):
if val:
if val =='all':
jobs = common._db.nJobs("list")
else:
jobs = self.parseRange_(val)
pass
else:
raise CrabException("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
pass
if (self.UseServer== 1):
from KillerServer import KillerServer
self.actions[opt] = KillerServer(self.cfg_params,jobs)
else:
from Killer import Killer
self.actions[opt] = Killer(self.cfg_params,jobs)
elif ( opt == '-stopWorkflow' ):
if self.UseServer==1 and int(self.automation)==1:
from WorkflowHandler import WorkflowHandler
self.actions[opt] = WorkflowHandler(self.cfg_params)
else:
raise CrabException("ERROR: you can use this command only if you are running an automated workflow using CRABSERVER")
pass
elif ( opt == '-getoutput' or opt == '-get'):
if val=='all' or val==None or val=='':
jobs = 'all'
else:
jobs = self.parseRange_(val)
if (self.UseServer== 1):
from GetOutputServer import GetOutputServer
self.actions[opt] = GetOutputServer(self.cfg_params,jobs)
else:
from GetOutput import GetOutput
self.actions[opt] = GetOutput(self.cfg_params,jobs)
elif ( opt == '-resubmit' ):
if val:
if val=='all':
jobs = common._db.nJobs('list')
if val=='bad':
jobs = 'bad'
else:
jobs = self.parseRange_(val)
if (self.UseServer== 1):
from ResubmitterServer import ResubmitterServer
self.actions[opt] = ResubmitterServer(self.cfg_params, jobs)
else:
from Resubmitter import Resubmitter
self.actions[opt] = Resubmitter(self.cfg_params, jobs)
else:
common.logger.info("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
common.logger.info("WARNING: _all_ job specified in the range will be resubmitted!!!")
pass
pass
elif ( opt == '-forceResubmit' ):
if val:
if val=='all':
jobs = common._db.nJobs('list')
else:
jobs = self.parseRange_(val)
if (self.UseServer== 1):
from ResubmitterServerForced import ResubmitterServerForced
self.actions[opt] = ResubmitterServerForced(self.cfg_params, jobs)
else:
from ResubmitterForced import ResubmitterForced
self.actions[opt] = ResubmitterForced(self.cfg_params, jobs)
else:
common.logger.info("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
common.logger.info("WARNING: _all_ job specified in the range will be resubmitted!!!")
pass
pass
elif ( opt in ['-testJdl','-listMatch', '-match']):
jobs = self.parseRange_(val)
if len(jobs) != 0:
# Instantiate Checker object
from Checker import Checker
self.actions[opt] = Checker(self.cfg_params, jobs)
elif ( opt == '-postMortem' ):
if val:
jobs = self.parseRange_(val)
if len( jobs ) > 1:
raise CrabException("Only single job id allowed for %s command!" % opt )
else:
raise CrabException("Warning: please specify a job id")
pass
if (self.UseServer== 1):
from PostMortemServer import PostMortemServer
self.actions[opt] = PostMortemServer(self.cfg_params,jobs)
else:
from PostMortem import PostMortem
self.actions[opt] = PostMortem(self.cfg_params, jobs)
elif ( opt == '-clean' ):
if val != None:
raise CrabException("No range allowed for '-clean'")
if (self.UseServer== 1):
from CleanerServer import CleanerServer
self.actions[opt] = CleanerServer(self.cfg_params)
else:
from Cleaner import Cleaner
self.actions[opt] = Cleaner(self.cfg_params)
elif ( opt in ['-printJdl','-createJdl']):
"""
Materialize JDL
"""
## Temporary:
if opt == '-printJdl':
common.logger.info("WARNING: -printJdl option is deprecated : please use -createJdl \n")
if val =='all' or val == None or val == '':
jobs = common._db.nJobs("list")
else:
jobs = self.parseRange_(val)
pass
from JdlWriter import JdlWriter
self.actions[opt] = JdlWriter(self.cfg_params, jobs)
elif ( opt == '-publish'):
from Publisher import Publisher
self.actions[opt] = Publisher(self.cfg_params)
### FEDE FOR PUBLICATION WITH NO INPUT FILES###
elif ( opt == '-publishNoInp'):
from Publisher import Publisher
self.cfg_params['USER.no_inp'] = 1
self.actions[opt] = Publisher(self.cfg_params)
elif ( opt == '-checkPublication' ):
from InspectDBS import InspectDBS
self.actions[opt] = InspectDBS(self.cfg_params)
elif ( opt == '-copyData' ):
if val =='all' or val == None or val == '':
jobs = common._db.nJobs("list")
else:
jobs = self.parseRange_(val)
if (self.UseServer== 1):
from StatusServer import StatusServer
status = StatusServer(self.cfg_params)
else:
# from Status import Status
# status = Status(self.cfg_params)
status=None
from CopyData import CopyData
self.actions[opt] = CopyData(self.cfg_params, jobs,status)
elif ( opt == '-validateCfg' ):
from ValidateCfg import ValidateCfg
config= {'pset' : self.cfg_params.get('CMSSW.pset','None')}
if val :
config['pset']=val
self.actions[opt] = ValidateCfg(config)
elif ( opt == '-renewCredential' ):
if (self.UseServer== 1):
from CredentialRenew import CredentialRenew
self.actions[opt] = CredentialRenew(self.cfg_params)
else:
msg = "The option [-renewProxy] can be used only with the server modality!"
raise CrabException(msg)
elif ( opt == '-report' ):
if (self.UseServer== 1):
from StatusServer import StatusServer
StatusServer(self.cfg_params).query(display=False)
else:
# cause a core dump....
#from Status import Status
#Status(self.cfg_params).query(display=False)
pass
from Reporter import Reporter
self.actions[opt] = Reporter(self.cfg_params)
elif ( opt == '-cleanCache' ):
from CacheCleaner import CacheCleaner
self.actions[opt] = CacheCleaner()
elif ( opt == '-uploadLog' ):
jobid = -1
jobs = self.parseRange_(val)
if len( jobs ) > 1:
common.logger.info("Only single job id allowed for %s command!" % opt )
elif len (jobs) == 1:
jobid = jobs[0]
from ReportUploader import ReportUploader
self.actions[opt] = ReportUploader( self.cfg_params, jobid )
pass
return
