#}}} # 2018{{{ if "2018" in args.years: for baby in babies_2018: for little_baby in little_babies(baby): command_list.append( './ttH%sLooper "%s" "RunII" "%s" "%s" "%s" "%s" "%s" "%s" "%s" "%s"' % (args.channel, args.selection, args.tag, args.bdt, args.bkg_options, little_baby, "2018", "_" + str(idx), syst, args.l1_prefire)) idx += 1 #}}} #}}} print "------------------------------------------------------------" nPar = 12 parallel_utils.submit_jobs(command_list, nPar) print "------------------------------------------------------------" # histograms, hadd, cleanup{{{ print "after parallel_utils.submit_jobs..." if args.babymaker: histos = glob.glob("MVABaby_ttH%s_%s_*.root" % (args.channel, args.tag)) else: histos = glob.glob("%s_%s_histogramsRunII_*.root" % (args.selection, args.tag)) good_histos = [] for hist in histos: size = os.stat(hist).st_size * (1. / (1024)) if size >= 1.: good_histos.append(hist) print("good hist: %s, size (kb): %d" % (hist, os.stat(hist).st_size * (1. / (1024))))
#parallel_utils.run('python plot_wrapper.py --input_file "../%s" --plot_type "std_2017" --plot_labels "FCNC Hadronic|Loose MVA Presel." --signals "TT_FCNC_hut|ST_FCNC_hut" --backgrounds "DiPhoton|QCD_GammaJets_imputed|TTGG|TTGJets|TTJets|VG"' % ("ttHHadronic_RunII_MVA_Presel_%s_histogramsRunII.root" % (args.tag + "_impute_hut_BDT_FCNC"))) #parallel_utils.run('python plot_wrapper.py --input_file "../%s" --plot_type "std_2017" --plot_labels "FCNC Hadronic|Loose MVA Presel." --signals "TT_FCNC_hct|ST_FCNC_hct" --backgrounds "DiPhoton|QCD_GammaJets_imputed|TTGG|TTGJets|TTJets|VG"' % ("ttHHadronic_RunII_MVA_Presel_%s_histogramsRunII.root" % (args.tag + "_impute_hct_BDT_FCNC"))) #os.chdir("../") #}}} >>>>>>> dd68e36d52dc32a9ea20698a1283183443a6d3e7 do_mvas = False if do_mvas: os.chdir("../MVAs/") command_list = [] command_list.append('python prep.py --input "../Loopers/MVABaby_ttHLeptonic_%s_FCNC.root" --channel "Leptonic" --fcnc_hut --tag "_hut"' % (args.tag)) command_list.append('python prep.py --input "../Loopers/MVABaby_ttHHadronic_%s_FCNC.root" --channel "Hadronic" --fcnc_hut --tag "_hut"' % (args.tag + "_impute")) command_list.append('python prep.py --input "../Loopers/MVABaby_ttHLeptonic_%s_FCNC.root" --channel "Leptonic" --fcnc_hct --tag "_hct"' % (args.tag)) command_list.append('python prep.py --input "../Loopers/MVABaby_ttHHadronic_%s_FCNC.root" --channel "Hadronic" --fcnc_hct --tag "_hct"' % (args.tag + "_impute")) parallel_utils.submit_jobs(command_list, 4) # MVA Training parallel_utils.run('python train.py --input "ttHLeptonic_%s_FCNC_features_hut.hdf5" --channel "Leptonic" --tag "%s" --ext ""' % (args.tag, args.tag + "_hut")) parallel_utils.run('python train.py --input "ttHHadronic_%s_FCNC_features_hut.hdf5" --channel "Hadronic" --tag "%s" --ext ""' % (args.tag + "_impute", args.tag + "_impute_hut")) parallel_utils.run('python train.py --input "ttHLeptonic_%s_FCNC_features_hct.hdf5" --channel "Leptonic" --tag "%s" --ext ""' % (args.tag, args.tag + "_hct")) parallel_utils.run('python train.py --input "ttHHadronic_%s_FCNC_features_hct.hdf5" --channel "Hadronic" --tag "%s" --ext ""' % (args.tag + "_impute", args.tag + "_impute_hct")) do_dnns = False if do_dnns: os.chdir("../MVAs/") command_list = [] #command_list.append('python prep_dnn.py --input "../Loopers/MVABaby_ttHLeptonic_%s_FCNC.root" --channel "Leptonic" --fcnc --z_score --signal "FCNC_hut" --backgrounds "ttH" --tag "hut_FCNC_vs_SMHiggs"' % (args.tag)) #command_list.append('python prep.py --input "../Loopers/MVABaby_ttHLeptonic_%s_FCNC.root" --channel "Leptonic" --fcnc_hut --FCNC_vs_SMHiggs --tag "_hut_FCNC_vs_SMHiggs"' % (args.tag))
command_list.append( 'python prep.py --input "../Loopers/MVABaby_ttHHadronic_%s_impute_FCNC.root" --channel "Hadronic" --signal "tt_fcnc_%s,st_fcnc_%s" --bkg "%s" --features "%s" --tag "addTopTaggers_%s"' % (args.tag, coupling.lower(), coupling.lower(), non_resonant_bkg + "," + sm_higgs, training_features_all, coupling.lower())) # Add top taggers non-resonant command_list.append( 'python prep.py --input "../Loopers/MVABaby_ttHHadronic_%s_impute_FCNC.root" --channel "Hadronic" --signal "tt_fcnc_%s,st_fcnc_%s" --bkg "%s" --features "%s" --tag "addTopTaggers_nonres_%s"' % (args.tag, coupling.lower(), coupling.lower(), non_resonant_bkg, training_features_all, coupling.lower())) # Add top taggers sm higgs command_list.append( 'python prep.py --input "../Loopers/MVABaby_ttHHadronic_%s_impute_FCNC.root" --channel "Hadronic" --signal "tt_fcnc_%s,st_fcnc_%s" --bkg "%s" --features "%s" --tag "addTopTaggers_smhiggs_%s"' % (args.tag, coupling.lower(), coupling.lower(), sm_higgs, training_features_all, coupling.lower())) parallel_utils.submit_jobs(command_list, 6) do_mvas = True if do_mvas: os.chdir("../MVAs/") #os.system("source ~/ttH/MVAs/setup.sh") command_list = [] for coupling in ["Hut", "Hct"]: # Baseline command_list.append( 'python train_bdt.py --input "ttHHadronic_%s_scale_diphoton_FCNC_features_baseline_%s.hdf5" --channel "Hadronic" --tag "baseline_%s_%s"' % (args.tag, coupling.lower(), args.tag, coupling.lower())) # Non-resonant command_list.append(
nPar = 1 command_list = [] for dir in dirs: if args.data_only: if "DoubleEG" not in dir: continue if args.no_signal: if "ttHJetToGG" in dir or "ttHToGG" in dir: continue files = glob.glob(dir + "/merged_ntuple*.root") name = dir.split("/")[-1] if not os.path.isdir(destination + name): os.system("mkdir %s" % destination + name) command = "addHistos %s %s %d %d" % (destination + name + "/merged_ntuple", dir + "/merged_ntuple", len(files), nPar) print(command) command_list.append(command) #print("addHistos %s %s %d %d" % (destination + name + "/merged_ntuple", dir + "/merged_ntuple", len(files), nPar)) #os.system("addHistos %s %s %d %d" % (destination + name + "/merged_ntuple", dir + "/merged_ntuple", len(files), nPar)) parallel_utils.submit_jobs(command_list, 8, False) #for dir in dirs: # Delete intermediate files #name = dir.split("/")[-1] #os.system("rm %s" % destination + name + "/merged_ntuple_*.root")
import parallel_utils scan_list = [ './ttHHadronicLooper "ttHHadronic_data_sideband_0b_train" "2017" "" "" "binned_NJets"', './ttHHadronicLooper "ttHHadronic_data_sideband_0b_test" "2017" "" "" "binned_NJets"', './ttHHadronicLooper "ttHHadronic_data_sideband_phoID_train" "2017" "" "" "binned_NJets"', './ttHHadronicLooper "ttHHadronic_data_sideband_phoID_test" "2017" "" "" "binned_NJets"', ] parallel_utils.submit_jobs(scan_list, 8)
print("[merge_skims.py] Merging skims from the following directories: ") for dir in directories: print(dir) command_list = [] for dir in directories: files = glob.glob(dir + "/*.root") target_dir = dir.replace(args.input_dir, args.output_dir) if not os.path.exists(target_dir): os.system("mkdir %s" % target_dir) file_size_map = {} total_size = 0 for file in files: size = get_size(file) file_size_map[file] = size total_size += size file_map = split_files(files, file_size_map, args.target_size) if args.debug > 0: print( "[merge_skims.py] Merging %d files of total size %.2f GB from directory %s into %d files in directory %s." % (len(files), size, dir, len(file_map.keys()), target_dir)) command_list += merge_files(file_map, args.output_dir) parallel_utils.submit_jobs(command_list, args.nCores)
import os, sys import parallel_utils from utils import * command_list = [] labels = label_map.keys() for label in labels: command = "python prep_data.py --label %s > log_%s.txt" % (label, label) command_list.append(command) parallel_utils.submit_jobs(command_list, len(labels))