def DBS2Publish(self, good_list):
####################################################
if self.no_inp == 1:
file_list = self.remove_input_from_fjr(good_list)
else:
file_list=good_list
print "file_list = ", file_list
####################################################
common.logger.log(10-1, "fjr with FrameworkJobReport Status='Success', file_list = "+str(file_list))
common.logger.log(10-1, "len(file_list) = "+str(len(file_list)))
if (len(file_list)>0):
BlocksList=[]
common.logger.info("--->>> Start dataset publication")
self.exit_status=self.publishDataset(file_list[0])
if (self.exit_status == '1'):
return self.exit_status
common.logger.info("--->>> End dataset publication")
common.logger.info("--->>> Start files publication")
for file in file_list:
Blocks=self.publishAJobReport(file,self.processedData)
if Blocks:
for x in Blocks: # do not allow multiple entries of the same block
if x not in BlocksList:
BlocksList.append(x)
# close the blocks
common.logger.log(10-1, "BlocksList = %s"%BlocksList)
dbswriter = DBSWriter(self.DBSURL)
for BlockName in BlocksList:
try:
closeBlock=dbswriter.manageFileBlock(BlockName,maxFiles= 1)
common.logger.log(10-1, "closeBlock %s"%closeBlock)
except DBSWriterError, ex:
common.logger.info("Close block error %s"%ex)
if (len(self.noEventsFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.noEventsFiles))+" published files contain 0 events are:")
for lfn in self.noEventsFiles:
common.logger.info("------ LFN: %s"%lfn)
if (len(self.noLFN)>0):
common.logger.info("--->>> WARNING: there are "+str(len(self.noLFN))+" files not published because they have empty LFN")
for pfn in self.noLFN:
common.logger.info("------ pfn: %s"%pfn)
if (len(self.problemFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.problemFiles))+" files not published because they had problem with copy to SE")
for lfn in self.problemFiles:
common.logger.info("------ LFN: %s"%lfn)
common.logger.info("--->>> End files publication")
#### FEDE for MULTI ####
for dataset_to_check in self.published_datasets:
self.cfg_params['USER.dataset_to_check']=dataset_to_check
from InspectDBS import InspectDBS
check=InspectDBS(self.cfg_params)
check.checkPublication()
#########################
return self.exit_status
def DBS2Publish(self, good_list):
####################################################
if self.no_inp == 1:
file_list = self.remove_input_from_fjr(good_list)
else:
file_list = good_list
print "file_list = ", file_list
####################################################
common.logger.log(
10 - 1,
"fjr with FrameworkJobReport Status='Success', file_list = " +
str(file_list))
common.logger.log(10 - 1, "len(file_list) = " + str(len(file_list)))
if (len(file_list) > 0):
BlocksList = []
common.logger.info("--->>> Start dataset publication")
self.exit_status = self.publishDataset(file_list[0])
if (self.exit_status == '1'):
return self.exit_status
common.logger.info("--->>> End dataset publication")
common.logger.info("--->>> Start files publication")
for file in file_list:
Blocks = self.publishAJobReport(file, self.processedData)
if Blocks:
for x in Blocks: # do not allow multiple entries of the same block
if x not in BlocksList:
BlocksList.append(x)
# close the blocks
common.logger.log(10 - 1, "BlocksList = %s" % BlocksList)
dbswriter = DBSWriter(self.DBSURL)
for BlockName in BlocksList:
try:
closeBlock = dbswriter.manageFileBlock(BlockName,
maxFiles=1)
common.logger.log(10 - 1, "closeBlock %s" % closeBlock)
except DBSWriterError, ex:
common.logger.info("Close block error %s" % ex)
if (len(self.noEventsFiles) > 0):
common.logger.info("--->>> WARNING: " +
str(len(self.noEventsFiles)) +
" published files contain 0 events are:")
for lfn in self.noEventsFiles:
common.logger.info("------ LFN: %s" % lfn)
if (len(self.noLFN) > 0):
common.logger.info(
"--->>> WARNING: there are " + str(len(self.noLFN)) +
" files not published because they have empty LFN")
for pfn in self.noLFN:
common.logger.info("------ pfn: %s" % pfn)
if (len(self.problemFiles) > 0):
common.logger.info(
"--->>> WARNING: " + str(len(self.problemFiles)) +
" files not published because they had problem with copy to SE"
)
for lfn in self.problemFiles:
common.logger.info("------ LFN: %s" % lfn)
common.logger.info("--->>> End files publication")
#### FEDE for MULTI ####
for dataset_to_check in self.published_datasets:
self.cfg_params['USER.dataset_to_check'] = dataset_to_check
from InspectDBS import InspectDBS
check = InspectDBS(self.cfg_params)
check.checkPublication()
#########################
return self.exit_status
def initializeActions_(self, opts):
"""
For each user action instantiate a corresponding
object and put it in the action dictionary.
"""
for opt in opts.keys():
val = opts[opt]
if ( opt == '-create' ):
if self.flag_continue:
msg = 'Cannot create an existing project. \n'
raise CrabException(msg)
if val and val != 'all':
msg = 'Per default, CRAB will create all jobs as specified in the crab.cfg file, not the command line!'
common.logger.info(msg)
msg = 'Submission will still take into account the number of jobs specified on the command line!\n'
common.logger.info(msg)
ncjobs = 'all'
#wmbs
if int(self.automation) == 1:
msg = 'Your jobs will be created on the server. Jobs informations will be known later.'
common.logger.info(msg)
# wmbs
from Requestor import Requestor
# Instantiate Creator object
self.creator = Requestor(self.job_type_name,
self.cfg_params,
ncjobs)
self.actions[opt] = self.creator
# Create and initialize JobList
common.job_list = JobList(1,self.creator.jobType())
else:
from Creator import Creator
# Instantiate Creator object
self.creator = Creator(self.job_type_name,
self.cfg_params,
ncjobs)
self.actions[opt] = self.creator
# create jobs in the DB
common._db.createJobs_(self.creator.nJobsL())
# Create and initialize JobList
common.job_list = JobList(common._db.nJobs(),
self.creator.jobType())
## jobs specs not needed if using WMBS
self.creator.writeJobsSpecsToDB()
taskinfo={}
taskinfo['cfgName'] = common.work_space.jobDir()+"/"+self.creator.jobType().configFilename()
taskinfo['dataset'] = self.cfg_params['CMSSW.datasetpath']
common.job_list.setScriptNames(self.job_type_name+'.sh')
common.job_list.setCfgNames(self.creator.jobType().configFilename())
common._db.updateTask_(taskinfo)
pass
elif ( opt == '-submit' ):
## Dealt with val == int so that -submit N means submit N jobs and not job # N
if (self.UseServer== 1):
from SubmitterServer import SubmitterServer
self.actions[opt] = SubmitterServer(self.cfg_params, self.parseRange_(val), val)
else:
from Submitter import Submitter
# Instantiate Submitter object
self.actions[opt] = Submitter(self.cfg_params, self.parseRange_(val), val)
# Create and initialize JobList
if len(common.job_list) == 0 :
common.job_list = JobList(common._db.nJobs(),
None)
pass
pass
elif ( opt == '-list' ):
'''
Print the relevant infos of a range-all jobs/task
'''
jobs = self.parseRange_(val)
common._db.dump(jobs)
pass
elif ( opt == '-printId' ):
'''
Print the unique name of the task if crab is used as client
Print the SID list of all the jobs
'''
#jid=False
#if val == 'full': jid=True
jid=True
common._db.queryID(self.UseServer,jid)
elif ( opt == '-status' ):
if (self.UseServer== 1):
from StatusServer import StatusServer
self.actions[opt] = StatusServer(self.cfg_params, val)
else:
from Status import Status
self.actions[opt] = Status(self.cfg_params, val)
elif ( opt == '-kill' ):
if val:
if val =='all':
jobs = common._db.nJobs("list")
else:
jobs = self.parseRange_(val)
pass
else:
raise CrabException("Warning: with '-kill' you _MUST_ specify a job range or 'all'")
pass
if (self.UseServer== 1):
from KillerServer import KillerServer
self.actions[opt] = KillerServer(self.cfg_params,jobs)
else:
from Killer import Killer
self.actions[opt] = Killer(self.cfg_params,jobs)
elif ( opt == '-stopWorkflow' ):
if self.UseServer==1 and int(self.automation)==1:
from WorkflowHandler import WorkflowHandler
self.actions[opt] = WorkflowHandler(self.cfg_params)
else:
raise CrabException("ERROR: you can use this command only if you are running an automated workflow using CRABSERVER")
pass
elif ( opt == '-getoutput' or opt == '-get'):
if val=='all' or val==None or val=='':
jobs = 'all'
else:
jobs = self.parseRange_(val)
if (self.UseServer== 1):
from GetOutputServer import GetOutputServer
self.actions[opt] = GetOutputServer(self.cfg_params,jobs)
else:
from GetOutput import GetOutput
self.actions[opt] = GetOutput(self.cfg_params,jobs)
elif ( opt == '-resubmit' ):
if val:
if val=='all':
jobs = common._db.nJobs('list')
if val=='bad':
jobs = 'bad'
else:
jobs = self.parseRange_(val)
if (self.UseServer== 1):
from ResubmitterServer import ResubmitterServer
self.actions[opt] = ResubmitterServer(self.cfg_params, jobs)
else:
from Resubmitter import Resubmitter
self.actions[opt] = Resubmitter(self.cfg_params, jobs)
else:
common.logger.info("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
common.logger.info("WARNING: _all_ job specified in the range will be resubmitted!!!")
pass
pass
elif ( opt == '-forceResubmit' ):
if val:
if val=='all':
jobs = common._db.nJobs('list')
else:
jobs = self.parseRange_(val)
if (self.UseServer== 1):
from ResubmitterServerForced import ResubmitterServerForced
self.actions[opt] = ResubmitterServerForced(self.cfg_params, jobs)
else:
from ResubmitterForced import ResubmitterForced
self.actions[opt] = ResubmitterForced(self.cfg_params, jobs)
else:
common.logger.info("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'")
common.logger.info("WARNING: _all_ job specified in the range will be resubmitted!!!")
pass
pass
elif ( opt in ['-testJdl','-listMatch', '-match']):
jobs = self.parseRange_(val)
if len(jobs) != 0:
# Instantiate Checker object
from Checker import Checker
self.actions[opt] = Checker(self.cfg_params, jobs)
elif ( opt == '-postMortem' ):
if val:
jobs = self.parseRange_(val)
if len( jobs ) > 1:
raise CrabException("Only single job id allowed for %s command!" % opt )
else:
raise CrabException("Warning: please specify a job id")
pass
if (self.UseServer== 1):
from PostMortemServer import PostMortemServer
self.actions[opt] = PostMortemServer(self.cfg_params,jobs)
else:
from PostMortem import PostMortem
self.actions[opt] = PostMortem(self.cfg_params, jobs)
elif ( opt == '-clean' ):
if val != None:
raise CrabException("No range allowed for '-clean'")
if (self.UseServer== 1):
from CleanerServer import CleanerServer
self.actions[opt] = CleanerServer(self.cfg_params)
else:
from Cleaner import Cleaner
self.actions[opt] = Cleaner(self.cfg_params)
elif ( opt in ['-printJdl','-createJdl']):
"""
Materialize JDL
"""
## Temporary:
if opt == '-printJdl':
common.logger.info("WARNING: -printJdl option is deprecated : please use -createJdl \n")
if val =='all' or val == None or val == '':
jobs = common._db.nJobs("list")
else:
jobs = self.parseRange_(val)
pass
from JdlWriter import JdlWriter
self.actions[opt] = JdlWriter(self.cfg_params, jobs)
elif ( opt == '-publish'):
from Publisher import Publisher
self.actions[opt] = Publisher(self.cfg_params)
### FEDE FOR PUBLICATION WITH NO INPUT FILES###
elif ( opt == '-publishNoInp'):
from Publisher import Publisher
self.cfg_params['USER.no_inp'] = 1
self.actions[opt] = Publisher(self.cfg_params)
elif ( opt == '-checkPublication' ):
from InspectDBS import InspectDBS
self.actions[opt] = InspectDBS(self.cfg_params)
elif ( opt == '-copyData' ):
if val =='all' or val == None or val == '':
jobs = common._db.nJobs("list")
else:
jobs = self.parseRange_(val)
if (self.UseServer== 1):
from StatusServer import StatusServer
status = StatusServer(self.cfg_params)
else:
# from Status import Status
# status = Status(self.cfg_params)
status=None
from CopyData import CopyData
self.actions[opt] = CopyData(self.cfg_params, jobs,status)
elif ( opt == '-validateCfg' ):
from ValidateCfg import ValidateCfg
config= {'pset' : self.cfg_params.get('CMSSW.pset','None')}
if val :
config['pset']=val
self.actions[opt] = ValidateCfg(config)
elif ( opt == '-renewCredential' ):
if (self.UseServer== 1):
from CredentialRenew import CredentialRenew
self.actions[opt] = CredentialRenew(self.cfg_params)
else:
msg = "The option [-renewProxy] can be used only with the server modality!"
raise CrabException(msg)
elif ( opt == '-report' ):
if (self.UseServer== 1):
from StatusServer import StatusServer
StatusServer(self.cfg_params).query(display=False)
else:
# cause a core dump....
#from Status import Status
#Status(self.cfg_params).query(display=False)
pass
from Reporter import Reporter
self.actions[opt] = Reporter(self.cfg_params)
elif ( opt == '-cleanCache' ):
from CacheCleaner import CacheCleaner
self.actions[opt] = CacheCleaner()
elif ( opt == '-uploadLog' ):
jobid = -1
jobs = self.parseRange_(val)
if len( jobs ) > 1:
common.logger.info("Only single job id allowed for %s command!" % opt )
elif len (jobs) == 1:
jobid = jobs[0]
from ReportUploader import ReportUploader
self.actions[opt] = ReportUploader( self.cfg_params, jobid )
pass
return
def run(self):
"""
parse of all xml file on res dir and creation of distionary
"""
task = common._db.getTask()
good_list=[]
for job in task.getJobs():
fjr = self.fjrDirectory + job['outputFiles'][-1]
if (job.runningJob['applicationReturnCode']!=0 or job.runningJob['wrapperReturnCode']!=0): continue
# get FJR filename
fjr = self.fjrDirectory + job['outputFiles'][-1]
reports = readJobReport(fjr)
if len(reports)>0:
if reports[0].status == "Success":
good_list.append(fjr)
####################################################
if self.no_inp == 1:
file_list = self.remove_input_from_fjr(good_list)
else:
file_list=good_list
print "file_list = ", file_list
####################################################
common.logger.log(10-1, "fjr with FrameworkJobReport Status='Success', file_list = "+str(file_list))
common.logger.log(10-1, "len(file_list) = "+str(len(file_list)))
if (len(file_list)>0):
BlocksList=[]
common.logger.info("--->>> Start dataset publication")
self.exit_status=self.publishDataset(file_list[0])
if (self.exit_status == '1'):
return self.exit_status
common.logger.info("--->>> End dataset publication")
common.logger.info("--->>> Start files publication")
for file in file_list:
Blocks=self.publishAJobReport(file,self.processedData)
if Blocks:
for x in Blocks: # do not allow multiple entries of the same block
if x not in BlocksList:
BlocksList.append(x)
# close the blocks
common.logger.log(10-1, "BlocksList = %s"%BlocksList)
dbswriter = DBSWriter(self.DBSURL)
for BlockName in BlocksList:
try:
closeBlock=dbswriter.manageFileBlock(BlockName,maxFiles= 1)
common.logger.log(10-1, "closeBlock %s"%closeBlock)
except DBSWriterError, ex:
common.logger.info("Close block error %s"%ex)
if (len(self.noEventsFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.noEventsFiles))+" published files contain 0 events are:")
for lfn in self.noEventsFiles:
common.logger.info("------ LFN: %s"%lfn)
if (len(self.noLFN)>0):
common.logger.info("--->>> WARNING: there are "+str(len(self.noLFN))+" files not published because they have empty LFN")
for pfn in self.noLFN:
common.logger.info("------ pfn: %s"%pfn)
if (len(self.problemFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.problemFiles))+" files not published because they had problem with copy to SE")
for lfn in self.problemFiles:
common.logger.info("------ LFN: %s"%lfn)
common.logger.info("--->>> End files publication")
#### FEDE for MULTI ####
for dataset_to_check in self.published_datasets:
self.cfg_params['USER.dataset_to_check']=dataset_to_check
from InspectDBS import InspectDBS
check=InspectDBS(self.cfg_params)
check.checkPublication()
#########################
return self.exit_status
