def getJobStatistics(LOGDIR,OUTFILE): DIR_SUMMARY = {} SUMMARY = {} ###stores full stat SINGLE_DIR_DATA = {} ###stores single entries MEAN_COMP = {} ### stores stat info firstDir = True tempI=0 samplesBinEdges = {} #{'lowerBin':{},'upperBin':{}} #Begin dir analysis if os.path.isdir(LOGDIR)==False and os.path.islink(LOGDIR)==False: print LOGDIR print "[ERROR] Directory not Valid" exit(1) SUMMARY = {"Success":0, "Failures":0, "Error":{}} spDirName = splitDirName(LOGDIR) if OUTFILE=="": OUTFILE = LOGDIR.strip('/').split("/")[-1]+'.root' print "---" print "OutFile:",OUTFILE print "Analyzing "+LOGDIR print "---" ###Parsing Files CRAB_LOGDIR = LOGDIR+"/res/" ### use stdout or xml for both? if options.TYPE == "CRAB": LOGS = os.popen("ls "+CRAB_LOGDIR+"*.stdout") elif options.TYPE == "CMSSW": LOGS = os.popen("ls "+LOGDIR+"/*.xml") elif options.TYPE == "CMSSWCRAB": LOGS = os.popen("ls "+CRAB_LOGDIR+"CMSSW*.stdout") else: printError("Please choose a type: CRAB, CMSSW or CMSSWCRAB") ### creating a TTree outFile = ROOT.TFile(OUTFILE,"RECREATE") perfTree = ROOT.TTree('Performance', 'Performance') ### loop over logfiles/jobs Total_jobs=0 Total_successfulJobs=0 valueContainer = {} for x in LOGS: #Parse crabDir x = x.strip('\n') if options.TYPE == "CRAB": rValue = parseDir_Crab(LOGDIR, x, acceptedSummaries) elif options.TYPE == "CMSSW": rValue = parseDir_CMSSW( x, acceptedSummaries) elif options.TYPE == "CMSSWCRAB": rValue = parseDir_CMSSWCrab( LOGDIR, x, 'cmssw', acceptedSummaries) if rValue!=1: job_output = rValue else: continue #end Parse crabDir Total_jobs+=1 if job_output["Success"]: Total_successfulJobs+=1 ###User Quantities, aka combination of logged quantities totalMB = 0 totalActualMB = 0 totalReadTime = 0 if job_output.has_key('tstoragefile-read-total-megabytes'): totalMB += float(job_output['tstoragefile-read-total-megabytes']) if job_output.has_key('tstoragefile-readv-total-megabytes'): totalMB += float(job_output['tstoragefile-readv-total-megabytes']) if job_output.has_key('tstoragefile-read-actual-total-megabytes'): totalActualMB += float(job_output['tstoragefile-read-actual-total-megabytes']) if job_output.has_key('tstoragefile-readv-actual-total-megabytes'): totalActualMB += float(job_output['tstoragefile-readv-actual-total-megabytes']) ### TODO: some estimate of read_kB/event ### TODO: find a good error handling computeErrorStat(job_output, SUMMARY) ### here, we are creating the tree structure, depending on the log content for label in job_output.keys(): if not valueContainer.has_key(label): if isinstance( job_output[label] , float) or isinstance( job_output[label] , list) : valueContainer[label]=ROOT.std.vector( float )() perfTree.Branch(str(makeCleanLabel(label,job_output)),valueContainer[str(label)]) if label=="Error": valueContainer[label]=ROOT.std.vector( float )() perfTree.Branch(str(makeCleanLabel(label,job_output)),valueContainer[str(label)]) # valueContainer[label+"_weight"]=ROOT.std.vector( float )() # perfTree.Branch(str(makeCleanLabel(label,job_output))+"_weight",valueContainer[str(label)]) pushJobOutput( label, job_output, SINGLE_DIR_DATA) isFirstJob=False ###end log cycle ### ###the previous loop gathers the info and creates the tree structure ###this one actually fills the tree ###TODO: better if conditions for job in range(0,Total_successfulJobs): for label in SINGLE_DIR_DATA.keys(): if label=="Error": continue if isinstance( SINGLE_DIR_DATA[label][job] , float): valueContainer[label].clear() valueContainer[label].push_back(SINGLE_DIR_DATA[label][job]) elif isinstance( SINGLE_DIR_DATA[label][job] , list): valueContainer[label].clear() for entry in SINGLE_DIR_DATA[label][job]: valueContainer[label].push_back(entry) ###tree filling perfTree.Fill() # Still need to record errors: label="Error" for err in SUMMARY[label].keys(): print str(err)+" : "+str(SUMMARY[label][err]) for job in range(0,int(SUMMARY[label][err])): valueContainer[label].clear() valueContainer[label].push_back(float(err)) perfTree.Fill() #############################################3 something entirely new here -- the job info! gROOT.ProcessLine( "struct MyStruct {\ Char_t site[100];\ Char_t dataset[200];\ Char_t sw[100];\ Char_t config[100];\ Char_t script[100];\ Char_t eventsJob[100];\ Char_t timestamp[100];\ Char_t comments[200];\ Char_t njobs[100];\ };" ); from ROOT import MyStruct, AddressOf; mystruct = MyStruct(); info_tree = ROOT.TTree("info_tree","info_tree"); info_tree.Branch("site",AddressOf(mystruct,'site'),'site/C'); info_tree.Branch("dataset",AddressOf(mystruct,'dataset'),'dataset/C'); info_tree.Branch("sw",AddressOf(mystruct,'sw'),'sw/C'); info_tree.Branch("config",AddressOf(mystruct,'config'),'config/C'); info_tree.Branch("comments",AddressOf(mystruct,'comments'),'comments/C'); info_tree.Branch("eventsJob",AddressOf(mystruct,'eventsJob'),'eventsJob/C'); info_tree.Branch("timestamp",AddressOf(mystruct,'timestamp'),'timestamp/C'); info_tree.Branch("njobs",AddressOf(mystruct,'njobs'),'njobs/C'); # fullstring=LOGDIR.strip('/').split("/")[-1].replace(".root","") # print "starting out with fullstring ",fullstring # bla_site = fullstring[0:fullstring.find("-")] # bla_rest = fullstring[fullstring.find("-")+1:len(fullstring)] # bla_script=bla_rest[0:bla_rest.find("-")] # bla_rest = bla_rest[bla_rest.find("-")+1:len(bla_rest)] # bla_nevents=int(bla_rest[0:bla_rest.find("-")]) # bla_rest = bla_rest[bla_rest.find("-")+1:len(bla_rest)] # bla_config=bla_rest[0:bla_rest.find("-")] # bla_rest = bla_rest[bla_rest.find("-")+1:len(bla_rest)] # bla_timestamp=bla_rest[0:bla_rest.find("__")] #mystruct.site="%s" % bla_site #mystruct.config="%s" % bla_config #mystruct.script="%s" % bla_script #mystruct.timestamp="%s" % bla_timestamp #mystruct.nevents="ne%s" % bla_nevents #mystruct.njobs="nj%s" % str(Total_jobs) mystruct.site="%s" % options.SITE mystruct.config="%s" % options.CONFIG mystruct.sw="%s" % options.SW mystruct.timestamp="%s" % options.TIMESTAMP mystruct.eventsJob="ne%s" % options.EVENTSJOB mystruct.njobs="nj%s" % str(Total_jobs) mystruct.comments="ne%s" % options.COMMENTS info_tree.Fill() #############################################3 /something entirely new here -- the job info! outFile.Write() outFile.Close()