def getJobStatistics(LOGDIR,OUTFILE):
DIR_SUMMARY = {}
SUMMARY = {} ###stores full stat
SINGLE_DIR_DATA = {} ###stores single entries
MEAN_COMP = {} ### stores stat info
firstDir = True
tempI=0
samplesBinEdges = {} #{'lowerBin':{},'upperBin':{}}
#Begin dir analysis
if os.path.isdir(LOGDIR)==False and os.path.islink(LOGDIR)==False:
print LOGDIR
print "[ERROR] Directory not Valid"
exit(1)
SUMMARY = {"Success":0, "Failures":0, "Error":{}}
spDirName = splitDirName(LOGDIR)
if OUTFILE=="": OUTFILE = LOGDIR.strip('/').split("/")[-1]+'.root'
print "---"
print "OutFile:",OUTFILE
print "Analyzing "+LOGDIR
print "---"
###Parsing Files
CRAB_LOGDIR = LOGDIR+"/res/"
### use stdout or xml for both?
if options.TYPE == "CRAB": LOGS = os.popen("ls "+CRAB_LOGDIR+"*.stdout")
elif options.TYPE == "CMSSW": LOGS = os.popen("ls "+LOGDIR+"/*.xml")
elif options.TYPE == "CMSSWCRAB": LOGS = os.popen("ls "+CRAB_LOGDIR+"CMSSW*.stdout")
else: printError("Please choose a type: CRAB, CMSSW or CMSSWCRAB")
### creating a TTree
outFile = ROOT.TFile(OUTFILE,"RECREATE")
perfTree = ROOT.TTree('Performance', 'Performance')
### loop over logfiles/jobs
Total_jobs=0
Total_successfulJobs=0
valueContainer = {}
for x in LOGS:
#Parse crabDir
x = x.strip('\n')
if options.TYPE == "CRAB": rValue = parseDir_Crab(LOGDIR, x, acceptedSummaries)
elif options.TYPE == "CMSSW": rValue = parseDir_CMSSW( x, acceptedSummaries)
elif options.TYPE == "CMSSWCRAB": rValue = parseDir_CMSSWCrab( LOGDIR, x, 'cmssw', acceptedSummaries)
if rValue!=1: job_output = rValue
else: continue
#end Parse crabDir
Total_jobs+=1
if job_output["Success"]:
Total_successfulJobs+=1
###User Quantities, aka combination of logged quantities
totalMB = 0
totalActualMB = 0
totalReadTime = 0
if job_output.has_key('tstoragefile-read-total-megabytes'): totalMB += float(job_output['tstoragefile-read-total-megabytes'])
if job_output.has_key('tstoragefile-readv-total-megabytes'): totalMB += float(job_output['tstoragefile-readv-total-megabytes'])
if job_output.has_key('tstoragefile-read-actual-total-megabytes'): totalActualMB += float(job_output['tstoragefile-read-actual-total-megabytes'])
if job_output.has_key('tstoragefile-readv-actual-total-megabytes'): totalActualMB += float(job_output['tstoragefile-readv-actual-total-megabytes'])
### TODO: some estimate of read_kB/event
### TODO: find a good error handling
computeErrorStat(job_output, SUMMARY)
### here, we are creating the tree structure, depending on the log content
for label in job_output.keys():
if not valueContainer.has_key(label):
if isinstance( job_output[label] , float) or isinstance( job_output[label] , list) :
valueContainer[label]=ROOT.std.vector( float )()
perfTree.Branch(str(makeCleanLabel(label,job_output)),valueContainer[str(label)])
if label=="Error":
valueContainer[label]=ROOT.std.vector( float )()
perfTree.Branch(str(makeCleanLabel(label,job_output)),valueContainer[str(label)])
# valueContainer[label+"_weight"]=ROOT.std.vector( float )()
# perfTree.Branch(str(makeCleanLabel(label,job_output))+"_weight",valueContainer[str(label)])
pushJobOutput( label, job_output, SINGLE_DIR_DATA)
isFirstJob=False
###end log cycle
###
###the previous loop gathers the info and creates the tree structure
###this one actually fills the tree
###TODO: better if conditions
for job in range(0,Total_successfulJobs):
for label in SINGLE_DIR_DATA.keys():
if label=="Error": continue
if isinstance( SINGLE_DIR_DATA[label][job] , float):
valueContainer[label].clear()
valueContainer[label].push_back(SINGLE_DIR_DATA[label][job])
elif isinstance( SINGLE_DIR_DATA[label][job] , list):
valueContainer[label].clear()
for entry in SINGLE_DIR_DATA[label][job]:
valueContainer[label].push_back(entry)
###tree filling
perfTree.Fill()
# Still need to record errors:
label="Error"
for err in SUMMARY[label].keys():
print str(err)+" : "+str(SUMMARY[label][err])
for job in range(0,int(SUMMARY[label][err])):
valueContainer[label].clear()
valueContainer[label].push_back(float(err))
perfTree.Fill()
#############################################3 something entirely new here -- the job info!
gROOT.ProcessLine(
"struct MyStruct {\
Char_t site[100];\
Char_t dataset[200];\
Char_t sw[100];\
Char_t config[100];\
Char_t script[100];\
Char_t eventsJob[100];\
Char_t timestamp[100];\
Char_t comments[200];\
Char_t njobs[100];\
};" );
from ROOT import MyStruct, AddressOf;
mystruct = MyStruct();
info_tree = ROOT.TTree("info_tree","info_tree");
info_tree.Branch("site",AddressOf(mystruct,'site'),'site/C');
info_tree.Branch("dataset",AddressOf(mystruct,'dataset'),'dataset/C');
info_tree.Branch("sw",AddressOf(mystruct,'sw'),'sw/C');
info_tree.Branch("config",AddressOf(mystruct,'config'),'config/C');
info_tree.Branch("comments",AddressOf(mystruct,'comments'),'comments/C');
info_tree.Branch("eventsJob",AddressOf(mystruct,'eventsJob'),'eventsJob/C');
info_tree.Branch("timestamp",AddressOf(mystruct,'timestamp'),'timestamp/C');
info_tree.Branch("njobs",AddressOf(mystruct,'njobs'),'njobs/C');
# fullstring=LOGDIR.strip('/').split("/")[-1].replace(".root","")
# print "starting out with fullstring ",fullstring
# bla_site = fullstring[0:fullstring.find("-")]
# bla_rest = fullstring[fullstring.find("-")+1:len(fullstring)]
# bla_script=bla_rest[0:bla_rest.find("-")]
# bla_rest = bla_rest[bla_rest.find("-")+1:len(bla_rest)]
# bla_nevents=int(bla_rest[0:bla_rest.find("-")])
# bla_rest = bla_rest[bla_rest.find("-")+1:len(bla_rest)]
# bla_config=bla_rest[0:bla_rest.find("-")]
# bla_rest = bla_rest[bla_rest.find("-")+1:len(bla_rest)]
# bla_timestamp=bla_rest[0:bla_rest.find("__")]
#mystruct.site="%s" % bla_site
#mystruct.config="%s" % bla_config
#mystruct.script="%s" % bla_script
#mystruct.timestamp="%s" % bla_timestamp
#mystruct.nevents="ne%s" % bla_nevents
#mystruct.njobs="nj%s" % str(Total_jobs)
mystruct.site="%s" % options.SITE
mystruct.config="%s" % options.CONFIG
mystruct.sw="%s" % options.SW
mystruct.timestamp="%s" % options.TIMESTAMP
mystruct.eventsJob="ne%s" % options.EVENTSJOB
mystruct.njobs="nj%s" % str(Total_jobs)
mystruct.comments="ne%s" % options.COMMENTS
info_tree.Fill()
#############################################3 /something entirely new here -- the job info!
outFile.Write()
outFile.Close()
