def makeProjection(h, pt_avg_bin, eta_bin):
''' Projects TH3D on a pt/eta bin
'''
# return if lower boundary > upper boundary
if pt_avg_bin[0] > pt_avg_bin[1]: return
if eta_bin[0] > eta_bin[1]: return
if pt_avg_bin[0] not in pt_avg_thresholds:
raise ValueError("pt bin %r does not align with thresholds %r" %
(pt_avg_bin, pt_avg_thresholds))
if pt_avg_bin[1] not in pt_avg_thresholds:
raise ValueError("pt bin %r does not align with thresholds %r" %
(pt_avg_bin, pt_avg_thresholds))
if eta_bin[0] not in eta_thresholds:
raise ValueError("eta bin %r does not align with thresholds %r" %
(eta_bin, eta_thresholds))
if eta_bin[1] not in eta_thresholds:
raise ValueError("eta bin %r does not align with thresholds %r" %
(eta_bin, eta_thresholds))
result = None
for i_eta in range(len(eta_thresholds) - 1):
eta_bin_ = tuple(eta_thresholds[i_eta:i_eta + 2])
# Is the current bin inside eta_bin?
if not (eta_bin_[0] >= eta_bin[0] and eta_bin_[1] <= eta_bin[1]):
continue
bin_y = h.GetYaxis().FindBin(0.5 * sum(eta_bin))
for i_pt_avg in range(len(pt_avg_thresholds) - 1):
pt_avg_bin_ = tuple(pt_avg_thresholds[i_pt_avg:i_pt_avg + 2])
# Is the current pt inside pt_bin?
if not (pt_avg_bin_[0] >= pt_avg_bin[0]
and pt_avg_bin_[1] <= pt_avg_bin[1]):
continue
bin_z = h.GetZaxis().FindBin(0.5 * sum(pt_avg_bin))
if result is None:
result = h.ProjectionX(str(uuid.uuid1()), bin_y, bin_y, bin_z,
bin_z)
else:
result.Add(
h.ProjectionX(str(uuid.uuid1()), bin_y, bin_y, bin_z,
bin_z))
logger.info("Added %r %r for %r %r", eta_bin_, pt_avg_bin_,
eta_bin, pt_avg_bin)
return result
def getCommands(line):
commands = []
split = None
try:
m = re.search(r"SPLIT[0-9][0-9]*", line)
split = int(m.group(0).replace('SPLIT', ''))
except:
pass
line = line.split('#')[0]
if line:
if split:
logger.info("Splitting in %i jobs", split)
for i in range(split):
commands.append(line + " --nJobs %i --job %i" % (split, i))
else:
commands.append(line)
return commands
def getCommands( line ):
commands = []
split = None
try:
m=re.search(r"SPLIT[0-9][0-9]*", line)
split=int(m.group(0).replace('SPLIT',''))
except:
pass
line = line.split('#')[0]
if line:
if split:
logger.info( "Splitting in %i jobs", split )
for i in range(split):
commands.append(line+" --nJobs %i --job %i"%( split, i ))
else:
commands.append(line)
return commands
def makeProjection( h, pt_avg_bin, eta_bin ):
''' Projects TH3D on a pt/eta bin
'''
# return if lower boundary > upper boundary
if pt_avg_bin[0]>pt_avg_bin[1]: return
if eta_bin[0]>eta_bin[1]: return
if pt_avg_bin[0] not in pt_avg_thresholds: raise ValueError( "pt bin %r does not align with thresholds %r" %( pt_avg_bin, pt_avg_thresholds ))
if pt_avg_bin[1] not in pt_avg_thresholds: raise ValueError( "pt bin %r does not align with thresholds %r" %( pt_avg_bin, pt_avg_thresholds ))
if eta_bin[0] not in eta_thresholds: raise ValueError( "eta bin %r does not align with thresholds %r" %( eta_bin, eta_thresholds ))
if eta_bin[1] not in eta_thresholds: raise ValueError( "eta bin %r does not align with thresholds %r" %( eta_bin, eta_thresholds ))
result = None
for i_eta in range(len(eta_thresholds)-1):
eta_bin_ = tuple(eta_thresholds[i_eta:i_eta+2])
# Is the current bin inside eta_bin?
if not (eta_bin_[0]>=eta_bin[0] and eta_bin_[1]<=eta_bin[1]) : continue
bin_y = h.GetYaxis().FindBin( 0.5*sum(eta_bin) )
for i_pt_avg in range(len(pt_avg_thresholds)-1):
pt_avg_bin_ = tuple(pt_avg_thresholds[i_pt_avg:i_pt_avg+2])
# Is the current pt inside pt_bin?
if not (pt_avg_bin_[0]>=pt_avg_bin[0] and pt_avg_bin_[1]<=pt_avg_bin[1]) : continue
bin_z = h.GetZaxis().FindBin( 0.5*sum(pt_avg_bin) )
if result is None:
result = h.ProjectionX(str(uuid.uuid1()) , bin_y, bin_y, bin_z, bin_z)
else:
result.Add( h.ProjectionX(str(uuid.uuid1()) , bin_y, bin_y, bin_z, bin_z))
logger.info( "Added %r %r for %r %r", eta_bin_, pt_avg_bin_, eta_bin, pt_avg_bin )
return result
sqrt(r_error_data**2 / r_mean_data**2 +
r_error_mc**2 / r_mean_mc**2))
for tg in [mc, data, ratio]:
for p in reversed(range(tg.GetN() + 1)):
if tg.GetX()[p] == 0 or tg.GetY()[p] == 0:
tg.RemovePoint(p)
obj_name = {
'mpf': 'MPF_CHS',
'ptbal': 'PtBal_CHS',
'mpfNoType1': 'MPF-notypeI_CHS'
}
logger.info("Reading %s", args.inputDir)
stuff = []
# Eta binned TGraphs vs. pt
for abs_eta_bin in all_abs_eta_bins:
ptll_vs_ptll_file = os.path.join(
args.inputDir, ('ptll_profile_ptll_for_eta_%4.3f_%4.3f' %
(abs_eta_bin)).replace('.', '') + '.root')
ptll_mc, ptll_data = getProfiles(ptll_vs_ptll_file)
for method in ['mpf', 'ptbal', 'mpfNoType1']:
for alpha in ['10', '15', '20', '30']:
response_file = os.path.join(
args.inputDir,
('r_%s_%s_a%s_profile_ptll_for_eta_%4.3f_%4.3f' %
colors = [ j+1 for j in range(0,9) ] + [ j+31 for j in range(9,18) ]
from JetMET.JEC.L2res.thresholds import pt_avg_thresholds, pt_avg_bins, eta_thresholds, abs_eta_thresholds
thresholds = [-1.2+x*2.4/96. for x in range(97)]
weightString = "weight"
results_file = os.path.join( plot_directory, 'results.pkl' )
h = {}
p = {}
if os.path.exists( results_file ) and not args.overwrite:
h, p = pickle.load( file (results_file ) )
logger.info( "Loaded %s", results_file )
else:
for var in [ "A", "B" ]:
h[var] = {}
p[var] = {}
for s in samples:
logger.info( "Make TH3D for sample %s and variable %s", s.name, var )
h[var][s.name] = ROOT.TH3D( "h_%s_%s"%( var, s.name), "h_%s_%s"%(var, s.name),\
len(thresholds) - 1, array.array('d', thresholds),
len(eta_thresholds)-1, array.array('d', eta_thresholds),
len(pt_avg_thresholds) - 1, array.array('d', pt_avg_thresholds)
)
weight_ = "("+s.selectionString+")*("+s.combineWithSampleWeight(weightString)+")"
varString_ = pt_binning_variable+":Jet_eta[probe_jet_index%s]:%s>>h_%s_%s"%( jer_postfix, var+jer_postfix, var, s.name )
#
# Logger
#
import JetMET.tools.logger as logger
import RootTools.core.logger as logger_rt
logger = logger.get_logger( args.logLevel, logFile = None)
logger_rt = logger_rt.get_logger(args.logLevel, logFile = None)
if args.small:
args.targetDir += "_small"
output_directory = os.path.join(skim_ntuple_directory, args.targetDir, "merged_RelValQCD_FlatPt_15_3000HS")
output_filename = os.path.join(output_directory, 'jets.root')
if not os.path.exists( output_directory ):
os.makedirs( output_directory )
logger.info( "Created output directory %s", output_directory )
samples = [ { 'prefix':name, 'sample':dataset } for name, dataset in \
[
("AB", QCD_flat_AB_v01),
("AC", QCD_flat_AC_v01),
("AC_2", QCD_flat_AC_2sigma_v01),
("AC_3", QCD_flat_AC_3sigma_v01),
("AC_4", QCD_flat_AC_4sigma_v01),
] ]
variables_firstsample = [ "evt/l", "run/I", "lumi/I" ]
variables_persample = [ "nVert/I", "genPt/F", "rawPt/F", "eta/F", "phi/F", "chHEF/F", "neHEF/F", "phEF/F", "eEF/F", "muEF/F", "HFHEF/F", "HFEMEF/F" ]
jet_vars = [ s.split('/')[0] for s in variables_persample]
elif args.maxFiles > 0:
maxN = args.maxFiles
else:
maxN = -1
sample = FWLiteSample.fromDAS(sample_name, args.sample, maxN=maxN, dbFile=None)
if args.maxFiles > 0:
sample.files = sample.files[:args.maxFiles]
output_directory = os.path.join(skim_ntuple_directory, args.targetDir,
sample.name)
output_filename = os.path.join(output_directory, sample.name + '.root')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
logger.info("Created output directory %s", output_directory)
products = {
# 'gp':{'type':'vector<reco::GenParticle>', 'label':("genParticles")},
# 'jets':{'type':'vector<reco::GenJet>', 'label':("ak4GenJets")},
# 'genMET':{'type':'vector<reco::GenMET>', 'label':("genMetTrue")},
'jets': {
'type': 'vector<pat::Jet>',
'label': ("slimmedJets")
},
'vertices': {
'type': 'vector<reco::Vertex>',
'label': ('offlineSlimmedPrimaryVertices')
},
}
from JetMET.JetCorrector.JetSmearer import JetSmearer
if __name__ == "__main__":
# Logging
import JetMET.tools.logger as logger
logger = logger.get_logger('DEBUG', logFile=None)
smearer_mc = JetSmearer("Spring16_25nsV10_MC", "AK4PFchs")
# Logging
#args = (400, 420, 2.4, 24 )
#logger.info( "Hybrid pt: '%r' -> '%r'" , args, smearer_mc.hybrid_correction(*args) )
args = (50, 2.5, 35)
logger.info("Resolution '%r' -> '%r'", args,
smearer_mc.get_jet_resolution(*args))
args = (50, 2.5, 0)
logger.info("Resolution '%r' -> '%r'", args,
smearer_mc.get_jet_resolution(*args))
# Segfault otherwise (no, destructor doesn't work)
smearer_mc.delete()
probe_jet_eta_string = "-Jet_eta[probe_jet_index]"
eta_string = "negeta"
eta_tex_string = "#eta<0"
elif args.etaSign == +1:
probe_jet_eta_string = "Jet_eta[probe_jet_index]"
eta_string = "poseta"
eta_tex_string = "#eta>0"
elif args.etaSign == 0:
probe_jet_eta_string = "abs(Jet_eta[probe_jet_index])"
eta_string = "alleta"
eta_tex_string = "(both endcaps)"
# kinematic selection on tag & probe jet
kinSelectionString = "%f<Jet_pt[tag_jet_index]&&Jet_pt[tag_jet_index]<%f " % (
tag_jet_bin[0], tag_jet_bin[1])
logger.info("Jet selection: %s", kinSelectionString)
data.addSelectionString("&&".join(c[1] for c in selection))
data.addSelectionString(kinSelectionString)
if args.small:
data.reduceFiles(to=1)
#colors = [ j+1 for j in range(0,9) ] + [ j+31 for j in range(9,18) ]
variableString = "1/sinh(%s)*sin(Jet_phi[probe_jet_index]):1/sinh(%s)*cos(Jet_phi[probe_jet_index])" % (
probe_jet_eta_string, probe_jet_eta_string)
weightString = "met_chsPt*weight*(%s>0)" % probe_jet_eta_string
logger.info("Get plot with %s, and weight %s", variableString, weightString)
h_data = data.get2DHistoFromDraw(variableString=variableString,
pt_avg_bins = [(pt_avg_thresholds[i], pt_avg_thresholds[i + 1])
for i in range(len(pt_avg_thresholds) - 1)]
alpha_values = [0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5]
alpha_ref_value = 0.3
response = {}
shape = {}
for alpha in alpha_values:
try:
filename = os.path.join(
user_plot_directory, args.input_directory, args.triggers, args.era,
'a%i' % (100 * alpha),
'response_%s_results.pkl' % ("fit" if args.useFit else "mean"))
response[alpha] = pickle.load(file(filename))
logger.info("Loaded file %s", filename)
except IOError:
logger.info("Could not load file %s", filename)
try:
filename = os.path.join(user_plot_directory, args.input_directory,
args.triggers, args.era, 'a%i' % (100 * alpha),
'results.pkl')
shape[alpha] = pickle.load(file(filename))[0]
logger.info("Loaded file %s", filename)
except IOError:
logger.info("Could not load file %s", filename)
# x ... A,B
# y ... eta
# z ... pt_avg
if args.etaSign == -1:
probe_jet_eta_cutstring = "-Jet_eta[probe_jet_index]"
eta_string = "negeta"
eta_tex_string = "-%4.3f #leq #eta < -%4.3f"% ( -args.etaBin[0], -args.etaBin[1] )
elif args.etaSign == +1:
probe_jet_eta_cutstring = "Jet_eta[probe_jet_index]"
eta_string = "poseta"
eta_tex_string = "%4.3f #leq #eta < %4.3f"% probe_jet_abs_eta_bin
elif args.etaSign == 0:
probe_jet_eta_cutstring = "abs(Jet_eta[probe_jet_index])"
eta_string = "abseta"
eta_tex_string = "%4.3f #leq |#eta| < %4.3f"% probe_jet_abs_eta_bin
# kinematic selection on tag & probe jet
kinSelectionString = "%i<pt_avg&&pt_avg<%i&& %s > %4.3f && %s < %4.3f"% ( tag_jet_bin[0], tag_jet_bin[1], probe_jet_eta_cutstring, probe_jet_abs_eta_bin[0], probe_jet_eta_cutstring, probe_jet_abs_eta_bin[1] )
logger.info( "Jet selection: %s", kinSelectionString )
selection = [
("btb", "cos(Jet_phi[tag_jet_index] - Jet_phi[probe_jet_index]) < cos(2.7)" ),
("a30", "alpha<0.3" ),
("tgb", "abs(Jet_eta[tag_jet_index])<1.3" ),
("failIdVeto", "Sum$(JetFailId_pt*(JetFailId_pt>30))<30" ),
("kinSelection", kinSelectionString )
]
for s in samples:
s.addSelectionString( "&&".join(c[1] for c in selection))
if args.small:
s.reduceFiles( to = 1 )
colors = [ j+1 for j in range(0,9) ] + [ j+31 for j in range(9,18) ]
if args.etaSign == -1:
probe_jet_eta_string = "-Jet_eta[probe_jet_index]"
eta_string = "negeta"
eta_tex_string = "#eta<0"
elif args.etaSign == +1:
probe_jet_eta_string = "Jet_eta[probe_jet_index]"
eta_string = "poseta"
eta_tex_string = "#eta>0"
elif args.etaSign == 0:
probe_jet_eta_string = "abs(Jet_eta[probe_jet_index])"
eta_string = "alleta"
eta_tex_string = "(both endcaps)"
# kinematic selection on tag & probe jet
kinSelectionString = "%f<Jet_pt[tag_jet_index]&&Jet_pt[tag_jet_index]<%f "% ( tag_jet_bin[0], tag_jet_bin[1] )
logger.info( "Jet selection: %s", kinSelectionString )
data.addSelectionString( "&&".join(c[1] for c in selection))
data.addSelectionString( kinSelectionString )
if args.small:
data.reduceFiles( to = 1 )
#colors = [ j+1 for j in range(0,9) ] + [ j+31 for j in range(9,18) ]
variableString = "1/sinh(%s)*sin(Jet_phi[probe_jet_index]):1/sinh(%s)*cos(Jet_phi[probe_jet_index])" % ( probe_jet_eta_string, probe_jet_eta_string )
weightString = "met_chsPt*weight*(%s>0)"%probe_jet_eta_string
logger.info( "Get plot with %s, and weight %s", variableString, weightString )
h_data = data.get2DHistoFromDraw(variableString = variableString, binning = [60, -0.3, 0.3, 60, -0.3, 0.3], weightString=weightString)
smearer_mc = JetSmearer("Spring16_25nsV6_MC", "AK4PFchs") if isMC else None # ICHEP version
if isMC:
from JetMET.tools.puReweighting import getReweightingFunction
puRW = getReweightingFunction(data="PU_Run2016_36000_XSecCentral", mc='Summer16')
puRWDown = getReweightingFunction(data="PU_Run2016_36000_XSecDown", mc='Summer16')
puRWUp = getReweightingFunction(data="PU_Run2016_36000_XSecUp", mc='Summer16')
if options.small: options.targetDir = os.path.join( options.targetDir, 'small' )
output_directory = os.path.join( options.targetDir, options.processingEra, options.skim, sample.name )
if os.path.exists(output_directory) and options.overwrite:
if options.nJobs > 1:
logger.warning( "NOT removing directory %s because nJobs = %i", output_directory, options.nJobs )
else:
logger.info( "Output directory %s exists. Deleting.", output_directory )
shutil.rmtree(output_directory)
try: #Avoid trouble with race conditions in multithreading
os.makedirs(output_directory)
logger.info( "Created output directory %s.", output_directory )
except:
pass
#branches to be kept for data and MC
branchKeepStrings_DATAMC = [\
"run", "lumi", "evt", "isData", "rho", "nVert",
"ecalBadCalibFilter*",
"met_*", "chsSumPt",
# "metNoHF_pt", "metNoHF_phi",
# "puppiMet_pt","puppiMet_phi","puppiMet_sumEt","puppiMet_rawPt","puppiMet_rawPhi","puppiMet_rawSumEt",
data.texName = "data (2 e)"
index = 1
#data = test_data #FIXME
#data.lumi=1
yields = {}
allPlots = {}
yields[args.mode] = {}
data_selection = [getFilterCut(positiveWeight=True), getLeptonSelection(args.mode)]
if runrange is not None:
data_selection.append( runrange )
logger.info( "Set data selectionstring: %s", "&&".join(data_selection) )
#data.setSelectionString( data_selection ) #FIXME
data.name = "data"
data.style = styles.errorStyle(ROOT.kBlack)
data.weight = weight_data
data.read_variables = [ "JetGood[pt/F,eta/F,phi/F,area/F,btagCSV/F,rawPt/F]" ]
#data.read_variables += [ "photon[pt/F,eta/F,phi/F]", "nPhotonGood/I" ]
lumi_scale = data.lumi/1000
if args.dy == 'DY_HT_LO':
DY_sample = DY_HT_LO
dy_legend_text = "DY(HT)"
elif args.dy == 'DYnJets':
DY_sample = DYnJets
texNames = {
'Offset_e_ch':"charged",
'Offset_e_nh':"neutrals",
'Offset_e_ph':"photons"
}
# Eta thresholds (need to be aligned with ntuple production step)
from JetMET.JEC.L1res.thresholds import offset_eta_thresholds
# rather awkward way to use TTree Draw
binning_int = [ len(offset_eta_thresholds) - 1, 0, len(offset_eta_thresholds) - 1 ]
histos = []
for i_offset, offset in enumerate(["Offset_e_ch", "Offset_e_nh", "Offset_e_ph"]):
logger.info( "Draw: Obtain offset histo %s for sample %s", offset, mc.name )
# make histo with x-axis being the fixed-size vector index Iteration$ that counts from 0 to size-1
h = mc.get1DHistoFromDraw("Iteration$", binning = binning_int, weightString = offset)
# make new histogram with same number of bins but proper eta thresholds
h_eta = ROOT.TH1D(h.GetName()+'_eta', h.GetTitle(), len(offset_eta_thresholds)-1, array.array('d', offset_eta_thresholds) )
for i in range( h.GetNbinsX() + 1 ):
h_eta.SetBinContent(i, h.GetBinContent( i ) )
h_eta.SetBinError(i, h.GetBinError( i ) )
# divide by nEvents
h_eta.Scale(1./h.GetEntries())
h_eta.legendText = texNames[offset] if texNames.has_key(offset) else offset
h_eta.style = styles.lineStyle(colors[i_offset], errors = True )
histos.append( [h_eta] )
#colors = [ j+1 for j in range(0,9) ] + [ j+31 for j in range(9,18) ]
from JetMET.JEC.L2res.thresholds import pt_avg_thresholds, pt_avg_bins, eta_thresholds
pt_avg_bins = [(pt_avg_thresholds[i], pt_avg_thresholds[i+1]) for i in range( len( pt_avg_thresholds ) -1 ) ]
alpha_values = [0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5]
alpha_ref_value = 0.3
response = {}
shape = {}
for alpha in alpha_values:
try:
filename = os.path.join( user_plot_directory, args.input_directory, args.triggers, args.era, 'a%i'%(100*alpha), 'response_%s_results.pkl' % ("fit" if args.useFit else "mean") )
response[alpha] = pickle.load( file( filename ))
logger.info( "Loaded file %s", filename )
except IOError:
logger.info( "Could not load file %s", filename )
try:
filename = os.path.join( user_plot_directory, args.input_directory, args.triggers, args.era, 'a%i'%(100*alpha), 'results.pkl' )
shape[alpha] = pickle.load( file( filename ))[0]
logger.info( "Loaded file %s", filename )
except IOError:
logger.info( "Could not load file %s", filename )
# x ... A,B
# y ... eta
# z ... pt_avg
def makeProjection( h, pt_avg_bin, eta_bin ):
''' Projects TH3D on a pt/eta bin
'''
def make_batch_job(batch_job_file, batch_job_title, batch_output_dir, command):
# If X509_USER_PROXY is set, use existing proxy.
if options.dpm:
if host == 'lxplus':
from JetMET.tools.user import cern_proxy_certificate
proxy_location = cern_proxy_certificate
else:
proxy_location = None
from RootTools.core.helpers import renew_proxy
proxy = renew_proxy(proxy_location)
logger.info("Using proxy certificate %s", proxy)
proxy_cmd = "export X509_USER_PROXY=%s" % proxy
else:
proxy_cmd = ""
import subprocess
if host == 'heplx':
template =\
"""\
#!/bin/sh
#SBATCH -J {batch_job_title}
#SBATCH -D {pwd}
#SBATCH -o {batch_output_dir}batch-test.%j.out
{proxy_cmd}
voms-proxy-info -all
eval \`"scram runtime -sh"\`
echo CMSSW_BASE: {cmssw_base}
echo Executing user command
echo "{command}"
{command}
voms-proxy-info -all
""".format(\
command = command,
cmssw_base = os.getenv("CMSSW_BASE"),
batch_output_dir = batch_output_dir,
batch_job_title = batch_job_title,
pwd = os.getenv("PWD"),
proxy_cmd = proxy_cmd
)
elif host == 'lxplus':
template =\
"""\
#!/bin/bash
export CMSSW_PROJECT_SRC={cmssw_base}/src
cd $CMSSW_PROJECT_SRC
eval `scramv1 ru -sh`
alias python={python_release}
which python
python --version
{proxy_cmd}
voms-proxy-info -all
echo CMSSW_BASE: $CMSSW_BASE
cd {pwd}
echo Executing user command while in $PWD
echo "{command}"
{command}
voms-proxy-info -all
""".format(\
command = command,
cmssw_base = os.getenv("CMSSW_BASE"),
#batch_output_dir = batch_output_dir,
#batch_job_title = batch_job_title,
pwd = os.getenv("PWD"),
proxy_cmd = proxy_cmd,
python_release = subprocess.check_output(['which', 'python']).rstrip(),
)
batch_job = file(batch_job_file, "w")
batch_job.write(template)
batch_job.close()
return
data. SetPoint(i_bin, pt_mean_data, r_mean_data)
data. SetPointError(i_bin, pt_error_data, r_error_data)
if r_mean_mc>0 and r_mean_data>0:
ratio.SetPoint(i_bin, pt_mean_mc, r_mean_data/r_mean_mc)
ratio.SetPointError(i_bin, pt_error_mc, r_mean_data/r_mean_mc*sqrt(r_error_data**2/r_mean_data**2 + r_error_mc**2/r_mean_mc**2) )
for tg in [mc, data, ratio]:
for p in reversed(range(tg.GetN()+1)):
if tg.GetX()[p]==0 or tg.GetY()[p]==0:
tg.RemovePoint(p)
obj_name = {'mpf':'MPF_CHS', 'ptbal':'PtBal_CHS', 'mpfNoType1':'MPF-notypeI_CHS'}
logger.info( "Reading %s", args.inputDir )
stuff = []
# Eta binned TGraphs vs. pt
for abs_eta_bin in all_abs_eta_bins:
ptll_vs_ptll_file = os.path.join( args.inputDir, ( 'ptll_profile_ptll_for_eta_%4.3f_%4.3f'%( abs_eta_bin ) ).replace( '.','')+'.root' )
ptll_mc, ptll_data = getProfiles(ptll_vs_ptll_file)
for method in ['mpf', 'ptbal', 'mpfNoType1']:
for alpha in ['10', '15', '20', '30']:
response_file = os.path.join( args.inputDir, ('r_%s_%s_a%s_profile_ptll_for_eta_%4.3f_%4.3f'%( ( method, args.version, alpha ) + abs_eta_bin )).replace( '.','')+'.root' )
r_mc, r_data = getProfiles(response_file)
n_bins = r_mc.GetNbinsX()
if args.etaSign == -1:
probe_jet_eta_cutstring = "-Jet_eta[probe_jet_index]"
eta_string = "negeta"
elif args.etaSign == +1:
probe_jet_eta_cutstring = "Jet_eta[probe_jet_index]"
eta_string = "poseta"
elif args.etaSign == 0:
probe_jet_eta_cutstring = "abs(Jet_eta[probe_jet_index])"
eta_string = "abseta"
# kinematic selection on tag & probe jet
kinSelectionString = "%f<Jet_pt[tag_jet_index]&&Jet_pt[tag_jet_index]<%f&& %s > %f && %s < %f" % (
tag_jet_bin[0], tag_jet_bin[1], probe_jet_eta_cutstring,
probe_jet_abs_eta_bin[0], probe_jet_eta_cutstring,
probe_jet_abs_eta_bin[1])
logger.info("Jet selection: %s", kinSelectionString)
for s in samples:
s.addSelectionString("&&".join(c[1] for c in selection))
s.addSelectionString(kinSelectionString)
if args.small:
s.reduceFiles(to=1)
colors = [j + 1 for j in range(0, 9)] + [j + 31 for j in range(9, 18)]
variableString = "Jet_%s" % args.observable
weightString = "weight"
if args.observable.endswith('EF'):
if args.fraction:
drawString = "%s[probe_jet_index]" % variableString
-args.etaBin[1])
elif args.etaSign == +1:
probe_jet_eta_cutstring = "Jet_eta[probe_jet_index]"
eta_string = "poseta"
eta_tex_string = "%4.3f #leq #eta < %4.3f" % probe_jet_abs_eta_bin
elif args.etaSign == 0:
probe_jet_eta_cutstring = "abs(Jet_eta[probe_jet_index])"
eta_string = "abseta"
eta_tex_string = "%4.3f #leq |#eta| < %4.3f" % probe_jet_abs_eta_bin
# kinematic selection on tag & probe jet
kinSelectionString = "%f<pt_avg&&pt_avg<%f&& %s > %f && %s < %f" % (
tag_jet_bin[0], tag_jet_bin[1], probe_jet_eta_cutstring,
probe_jet_abs_eta_bin[0], probe_jet_eta_cutstring,
probe_jet_abs_eta_bin[1])
logger.info("Jet selection: %s", kinSelectionString)
selection = [
("btb",
"cos(Jet_phi[tag_jet_index] - Jet_phi[probe_jet_index]) < cos(2.7)"),
("a30", "alpha<0.3"), ("tgb", "abs(Jet_eta[tag_jet_index])<1.3"),
("failIdVeto", "Sum$(JetFailId_pt*(JetFailId_pt>30))<30"),
("kinSelection", kinSelectionString)
]
if args.small:
mc.files = [
'/afs/hephy.at/data/rschoefbeck02/postProcessed/L2res/v8/default/QCD_Pt_470to600/QCD_Pt_470to600_0.root'
]
#mc.reduceFiles( to = 1 )
pass
line = line.split('#')[0]
if line:
if split:
logger.info( "Splitting in %i jobs", split )
for i in range(split):
commands.append(line+" --nJobs %i --job %i"%( split, i ))
else:
commands.append(line)
return commands
if __name__ == '__main__':
if not len(args) == 1:
raise Exception("Only one argument accepted! Instead this was given: %s"%args)
if os.path.isfile(args[0]):
logger.info( "Reading commands from file: %s", args[0] )
commands = []
with open(args[0]) as f:
for line in f.xreadlines():
commands.extend( getCommands( line.rstrip("\n") ) )
elif type(args[0]) == type(""):
commands = getCommands( args[0] )
if commands:
logger.info( "Working on host %s", host )
if host == 'heplx':
if not os.path.isdir(batch_output_dir):
os.mkdir(batch_output_dir)
logger.info( "Batch system output file to be written to directory: %s", batch_output_dir )
logger = logger.get_logger(args.logLevel, logFile=None)
logger_rt = logger_rt.get_logger(args.logLevel, logFile=None)
if args.small:
args.targetDir += "_small"
sample_name = args.sample.lstrip('/').replace('/', '_')
if args.small:
maxN = 1
elif args.maxFiles > 0:
maxN = args.maxFiles
else:
maxN = -1
dbFile = os.path.join(cache_directory, 'JME_fwlite_cache.db')
logger.info("Using sample cache %s", dbFile)
sample = FWLiteSample.fromDAS(sample_name,
args.sample,
maxN=maxN,
dbFile=dbFile)
output_directory = os.path.join(skim_ntuple_directory, args.targetDir,
sample.name)
# Run only job number "args.job" from total of "args.nJobs"
if args.nJobs > 1:
n_files_before = len(sample.files)
sample = sample.split(args.nJobs)[args.job]
n_files_after = len(sample.files)
logger.info("Running job %i/%i over %i files from a total of %i.",
args.job, args.nJobs, n_files_after, n_files_before)
def make_batch_job( batch_job_file, batch_job_title, batch_output_dir , command ):
# If X509_USER_PROXY is set, use existing proxy.
if options.dpm:
if host == 'lxplus':
from JetMET.tools.user import cern_proxy_certificate
proxy_location = cern_proxy_certificate
else:
proxy_location = None
from RootTools.core.helpers import renew_proxy
proxy = renew_proxy( proxy_location )
logger.info( "Using proxy certificate %s", proxy )
proxy_cmd = "export X509_USER_PROXY=%s"%proxy
else:
proxy_cmd = ""
import subprocess
if host == 'heplx':
template =\
"""\
#!/bin/sh
#SBATCH -J {batch_job_title}
#SBATCH -D {pwd}
#SBATCH -o {batch_output_dir}batch-test.%j.out
{proxy_cmd}
voms-proxy-info -all
eval \`"scram runtime -sh"\`
echo CMSSW_BASE: {cmssw_base}
echo Executing user command
echo "{command}"
{command}
voms-proxy-info -all
""".format(\
command = command,
cmssw_base = os.getenv("CMSSW_BASE"),
batch_output_dir = batch_output_dir,
batch_job_title = batch_job_title,
pwd = os.getenv("PWD"),
proxy_cmd = proxy_cmd
)
elif host == 'lxplus':
template =\
"""\
#!/bin/bash
export CMSSW_PROJECT_SRC={cmssw_base}/src
cd $CMSSW_PROJECT_SRC
eval `scramv1 ru -sh`
alias python={python_release}
which python
python --version
{proxy_cmd}
voms-proxy-info -all
echo CMSSW_BASE: $CMSSW_BASE
cd {pwd}
echo Executing user command while in $PWD
echo "{command}"
{command}
voms-proxy-info -all
""".format(\
command = command,
cmssw_base = os.getenv("CMSSW_BASE"),
#batch_output_dir = batch_output_dir,
#batch_job_title = batch_job_title,
pwd = os.getenv("PWD"),
proxy_cmd = proxy_cmd,
python_release = subprocess.check_output(['which', 'python']).rstrip(),
)
batch_job = file(batch_job_file, "w")
batch_job.write(template)
batch_job.close()
return
colors = [j + 1 for j in range(0, 9)] + [j + 31 for j in range(9, 18)]
from JetMET.JEC.L2res.thresholds import pt_avg_thresholds, pt_avg_bins, eta_thresholds, abs_eta_thresholds
thresholds = [-1.2 + x * 2.4 / 96. for x in range(97)]
weightString = "weight"
results_file = os.path.join(plot_directory, 'results.pkl')
h = {}
p = {}
if os.path.exists(results_file) and not args.overwrite:
h, p = pickle.load(file(results_file))
logger.info("Loaded %s", results_file)
else:
for var in ["A", "B"]:
h[var] = {}
p[var] = {}
for s in samples:
logger.info("Make TH3D for sample %s and variable %s", s.name, var)
h[var][s.name] = ROOT.TH3D( "h_%s_%s"%( var, s.name), "h_%s_%s"%(var, s.name),\
len(thresholds) - 1, array.array('d', thresholds),
len(eta_thresholds)-1, array.array('d', eta_thresholds),
len(pt_avg_thresholds) - 1, array.array('d', pt_avg_thresholds)
)
weight_ = "(" + s.selectionString + ")*(" + s.combineWithSampleWeight(
weightString) + ")"
varString_ = pt_binning_variable + ":Jet_eta[probe_jet_index%s]:%s>>h_%s_%s" % (
from JetMET.JetCorrector.JetSmearer import JetSmearer
if __name__ == "__main__":
# Logging
import JetMET.tools.logger as logger
logger = logger.get_logger('DEBUG', logFile = None)
smearer_mc = JetSmearer("Spring16_25nsV10_MC", "AK4PFchs")
# Logging
#args = (400, 420, 2.4, 24 )
#logger.info( "Hybrid pt: '%r' -> '%r'" , args, smearer_mc.hybrid_correction(*args) )
args = (50, 2.5, 35 )
logger.info( "Resolution '%r' -> '%r'" , args, smearer_mc.get_jet_resolution(*args) )
args = (50, 2.5, 0 )
logger.info( "Resolution '%r' -> '%r'" , args, smearer_mc.get_jet_resolution(*args) )
# Segfault otherwise (no, destructor doesn't work)
smearer_mc.delete()
}
# Eta thresholds (need to be aligned with ntuple production step)
from JetMET.JEC.L1res.thresholds import offset_eta_thresholds
# rather awkward way to use TTree Draw
binning_int = [
len(offset_eta_thresholds) - 1, 0,
len(offset_eta_thresholds) - 1
]
histos = []
for i_offset, offset in enumerate(
["Offset_e_ch", "Offset_e_nh", "Offset_e_ph"]):
logger.info("Draw: Obtain offset histo %s for sample %s", offset, mc.name)
# make histo with x-axis being the fixed-size vector index Iteration$ that counts from 0 to size-1
h = mc.get1DHistoFromDraw("Iteration$",
binning=binning_int,
weightString=offset)
# make new histogram with same number of bins but proper eta thresholds
h_eta = ROOT.TH1D(h.GetName() + '_eta', h.GetTitle(),
len(offset_eta_thresholds) - 1,
array.array('d', offset_eta_thresholds))
for i in range(h.GetNbinsX() + 1):
h_eta.SetBinContent(i, h.GetBinContent(i))
h_eta.SetBinError(i, h.GetBinError(i))
# divide by nEvents
#data = test_data #FIXME
#data.lumi=1
yields = {}
allPlots = {}
yields[args.mode] = {}
data_selection = [
getFilterCut(positiveWeight=True),
getLeptonSelection(args.mode)
]
if runrange is not None:
data_selection.append(runrange)
logger.info("Set data selectionstring: %s", "&&".join(data_selection))
#data.setSelectionString( data_selection ) #FIXME
data.name = "data"
data.style = styles.errorStyle(ROOT.kBlack)
data.weight = weight_data
data.read_variables = ["JetGood[pt/F,eta/F,phi/F,area/F,btagCSV/F,rawPt/F]"]
#data.read_variables += [ "photon[pt/F,eta/F,phi/F]", "nPhotonGood/I" ]
lumi_scale = data.lumi / 1000
if args.dy == 'DY_HT_LO':
DY_sample = DY_HT_LO
dy_legend_text = "DY(HT)"
elif args.dy == 'DYnJets':
DY_sample = DYnJets
#
# Logger
#
import JetMET.tools.logger as logger
import RootTools.core.logger as logger_rt
logger = logger.get_logger( args.logLevel, logFile = None)
logger_rt = logger_rt.get_logger(args.logLevel, logFile = None)
if args.small:
args.targetDir += "_small"
output_directory = os.path.join(skim_ntuple_directory, args.targetDir, "ECAL_merged")
output_filename = os.path.join(output_directory, 'jets.root')
if not os.path.exists( output_directory ):
os.makedirs( output_directory )
logger.info( "Created output directory %s", output_directory )
samples = [ { 'prefix':name, 'fwlite':FWLiteSample.fromDAS( name, dataset, dbFile = os.path.join( cache_directory, 'fwlite_cache.db' )) } for name, dataset in \
[
( "GTv2_SRPFon_noPU", "/RelValQCD_FlatPt_15_3000HS_13UP17/CMSSW_9_2_9-92X_upgrade2017_realistic_Candidate_forECALStudies_HS1M_PF17-v1/MINIAODSIM" ),
( "GTv2_SRPFon_PUpmx25ns", "/RelValQCD_FlatPt_15_3000HS_13UP17/CMSSW_9_2_9-PUpmx25ns_92X_upgrade2017_realistic_Candidate_forECALStudies_HS1M_PF17-v1/MINIAODSIM" ),
( "GTv1_SRPFoff_noPU", "/RelValQCD_FlatPt_15_3000HS_13UP17/CMSSW_9_2_9-92X_upgrade2017_realistic_v10_HS1M_PF16-v1/MINIAODSIM" ),
( "GTv1_SRPFoff_PUpmx25ns", "/RelValQCD_FlatPt_15_3000HS_13UP17/CMSSW_9_2_9-PUpmx25ns_92X_upgrade2017_realistic_v10_HS1M_PF16-v1/MINIAODSIM" ),
( "GTv2_SRPFoff_noPU", "/RelValQCD_FlatPt_15_3000HS_13UP17/CMSSW_9_2_9-92X_upgrade2017_realistic_Candidate_forECALStudies_HS1M_PF16-v1/MINIAODSIM" ),
( "GTv2_SRPFoff_PUpmx25ns", "/RelValQCD_FlatPt_15_3000HS_13UP17/CMSSW_9_2_9-PUpmx25ns_92X_upgrade2017_realistic_Candidate_forECALStudies_HS1M_PF16-v1/MINIAODSIM" ),
] ]
products = {
# 'gp':{'type':'vector<reco::GenParticle>', 'label':("genParticles")},
# 'jets':{'type':'vector<reco::GenJet>', 'label':("ak4GenJets")},
# 'genMET':{'type':'vector<reco::GenMET>', 'label':("genMetTrue")},
## dphi(2,3) vs. probe jet eta
#weightString = "weight&&(third_jet_index>0)"
#variableString = "cos(Jet_phi[probe_jet_index] - Jet_phi[third_jet_index]):Jet_eta[probe_jet_index]"
#binning = [52, -5.2, 5.2, 40, -1, 1]
#logger.info( "Get plot with %s, and weight '%s' and sample selection %s", variableString, weightString, sample.selectionString )
#h = sample.get2DHistoFromDraw(variableString = variableString, binning = binning, weightString=weightString)
#plot = Plot2D.fromHisto( name = ( '2D_pt_%s_dPhiJet23_vs_eta_probe_jet'%args.pt ),
# histos = [[ h ]], texY = "cos(#Delta#phi(j_{2},j_{3}))", texX = "#eta(probe jet)"
# )
#draw2DPlots( [plot], 1.)
# minDphi(ETmiss,1,2,3) vs. probe jet eta
weightString = "weight"
variableString = "MaxIf$(abs(cos(met_chsPhi - Jet_phi)), Jet_pt>30&&Iteration$<5)"
binning = [ 40, 0, 1]
logger.info( "Get plot with %s, and weight '%s' and sample selection %s", variableString, weightString, sample.selectionString )
h = sample.get1DHistoFromDraw(variableString = variableString, binning = binning, weightString=weightString)
plot = Plot.fromHisto( name = ( '1D_pt_%s_maxCosdPhiMETJets'%args.pt ),
histos = [[ h ]], texX = "max cos(|#Delta#phi(MET, jets)|)", texY = "Events"
)
draw1DPlots( [plot], 1.)
weightString = "weight"
variableString = "MaxIf$(abs(cos(met_chsPhi - Jet_phi)), Jet_pt>30&&Iteration$<5):Jet_eta[probe_jet_index]"
binning = [52, -5.2, 5.2, 40, 0, 1]
logger.info( "Get plot with %s, and weight '%s' and sample selection %s", variableString, weightString, sample.selectionString )
h2D = sample.get2DHistoFromDraw(variableString = variableString, binning = binning, weightString=weightString)
plot2D = Plot2D.fromHisto( name = ( '2D_pt_%s_maxCosdPhiMETJets_vs_eta_probe_jet'%args.pt ),
histos = [[ h2D ]], texY = "max cos(|#Delta#phi(MET, jets)|)", texX = "#eta(probe jet)"
)
draw2DPlots( [plot2D], 1.)
isMC = not isData
if isMC:
from JetMET.tools.puReweighting import getReweightingFunction
puRW = getReweightingFunction(data="PU_Run2016_36000_XSecCentral", mc='Summer16')
puRWDown = getReweightingFunction(data="PU_Run2016_36000_XSecDown", mc='Summer16')
puRWUp = getReweightingFunction(data="PU_Run2016_36000_XSecUp", mc='Summer16')
if options.small: options.targetDir+='_small'
output_directory = os.path.join( options.targetDir, options.processingEra, options.skim, sample.name )
if os.path.exists(output_directory) and options.overwrite:
if options.nJobs > 1:
logger.warning( "NOT removing directory %s because nJobs = %i", output_directory, options.nJobs )
else:
logger.info( "Output directory %s exists. Deleting.", output_directory )
shutil.rmtree(output_directory)
try: #Avoid trouble with race conditions in multithreading
os.makedirs(output_directory)
logger.info( "Created output directory %s.", output_directory )
except:
pass
#branches to be kept for data and MC
branchKeepStrings_DATAMC = [\
"run", "lumi", "evt", "isData", "rho", "nVert",
"met_*", "chsSumPt",
# "metNoHF_pt", "metNoHF_phi",
# "puppiMet_pt","puppiMet_phi","puppiMet_sumEt","puppiMet_rawPt","puppiMet_rawPhi","puppiMet_rawSumEt",
"Flag_*","HLT_*",
if line:
if split:
logger.info("Splitting in %i jobs", split)
for i in range(split):
commands.append(line + " --nJobs %i --job %i" % (split, i))
else:
commands.append(line)
return commands
if __name__ == '__main__':
if not len(args) == 1:
raise Exception(
"Only one argument accepted! Instead this was given: %s" % args)
if os.path.isfile(args[0]):
logger.info("Reading commands from file: %s", args[0])
commands = []
with open(args[0]) as f:
for line in f.xreadlines():
commands.extend(getCommands(line.rstrip("\n")))
elif type(args[0]) == type(""):
commands = getCommands(args[0])
if commands:
logger.info("Working on host %s", host)
if host == 'heplx':
if not os.path.isdir(batch_output_dir):
os.mkdir(batch_output_dir)
logger.info(
"Batch system output file to be written to directory: %s",
logger = logger.get_logger( args.logLevel, logFile = None)
logger_rt = logger_rt.get_logger(args.logLevel, logFile = None)
if args.small:
args.targetDir += "_small"
sample_name = args.sample.lstrip('/').replace('/','_')
if args.small:
maxN = 1
elif args.maxFiles>0:
maxN = args.maxFiles
else:
maxN = -1
dbFile = os.path.join( cache_directory, 'JME_fwlite_cache.db' )
logger.info( "Using sample cache %s", dbFile )
sample = FWLiteSample.fromDAS( sample_name, args.sample, maxN = maxN, dbFile = dbFile)
output_directory = os.path.join(skim_ntuple_directory, args.targetDir, sample.name)
# Run only job number "args.job" from total of "args.nJobs"
if args.nJobs>1:
n_files_before = len(sample.files)
sample = sample.split(args.nJobs)[args.job]
n_files_after = len(sample.files)
logger.info( "Running job %i/%i over %i files from a total of %i.", args.job, args.nJobs, n_files_after, n_files_before)
output_filename = os.path.join(output_directory, sample.name + '.root')
if 'Run2018' in sample.name:
from FWCore.PythonUtilities.LumiList import LumiList
json = '$CMSSW_BASE/src/JetMET/diagnosis/python/pu2018/Cert_314472-325175_13TeV_PromptReco_Collisions18_JSON.txt'
if args.etaSign == -1:
probe_jet_eta_cutstring = "-Jet_eta[probe_jet_index]"
eta_string = "negeta"
eta_tex_string = "-%4.3f #leq #eta < -%4.3f"% ( -args.etaBin[0], -args.etaBin[1] )
elif args.etaSign == +1:
probe_jet_eta_cutstring = "Jet_eta[probe_jet_index]"
eta_string = "poseta"
eta_tex_string = "%4.3f #leq #eta < %4.3f"% probe_jet_abs_eta_bin
elif args.etaSign == 0:
probe_jet_eta_cutstring = "abs(Jet_eta[probe_jet_index])"
eta_string = "abseta"
eta_tex_string = "%4.3f #leq |#eta| < %4.3f"% probe_jet_abs_eta_bin
# kinematic selection on tag & probe jet
kinSelectionString = "%f<pt_avg&&pt_avg<%f&& %s > %f && %s < %f"% ( tag_jet_bin[0], tag_jet_bin[1], probe_jet_eta_cutstring, probe_jet_abs_eta_bin[0], probe_jet_eta_cutstring, probe_jet_abs_eta_bin[1] )
logger.info( "Jet selection: %s", kinSelectionString )
selection = [
("btb", "cos(Jet_phi[tag_jet_index] - Jet_phi[probe_jet_index]) < cos(2.7)" ),
("a30", "alpha<0.3" ),
("tgb", "abs(Jet_eta[tag_jet_index])<1.3" ),
("failIdVeto", "Sum$(JetFailId_pt*(JetFailId_pt>30))<30" ),
("kinSelection", kinSelectionString )
]
if args.small:
mc.files = [ '/afs/hephy.at/data/rschoefbeck02/postProcessed/L2res/v8/default/QCD_Pt_470to600/QCD_Pt_470to600_0.root' ]
#mc.reduceFiles( to = 1 )
colors = [ j+1 for j in range(0,9) ] + [ j+31 for j in range(9,18) ]
