예제 #1
0
fileParts = [submitFile]
sc = SampleCollection()
datasets = []

if options.dataCollections:
    print sc.sampleCollectionList()
    exit(0)

if options.datasets:
    datasets = options.datasets.split(',')
else:
    print "No dataset pecified"
    exit(0)

lumis = sc.sampleCollectionLumiList()
lumi = sc.getFixedLumi()
if options.refLumi != None:
    lumi = lumis[options.refLumi]
    print "Normalizing to %s pb-1" % (lumi)

for ds in datasets:
    ds = ds.strip()

    for s, n in sc.sampleList(ds):
        print n
        f = open(s)
        if not f == None:
            count = 0
            for l in f:
                if '.root' in l and not 'failed' in l:
                    count = count + 1
예제 #2
0
        if options.dataCollectionslong:
            sys.stdout.write("\t")
            for sample in sl:
                sys.stdout.write("%s  " % sample[1])
            print ""
            print ""
    exit(0)

if options.datasets:
    datasets = options.datasets.split(',')
else:
    print "No dataset specified"
    exit(0)

lumis = sc.sampleCollectionLumiList()
lumi = sc.getFixedLumi()
if options.refLumi != None:
    lumi = lumis[options.refLumi]
    print "Normalizing to %s pb-1" % (lumi)

for ds in datasets:
    ds = ds.strip()

    print ds
    # s: file, n:name, e:nEvts
    for s, n, e in sc.sampleList(ds):
        print "\t%s" % n
        #print "\t{0} {1} {2}".format(s, n, e)
        try:
            f = open(s)
        except IOError:
예제 #3
0
def submit(datasets,
           refLumi,
           era,
           numfile=5,
           noSubmit=False,
           verbose=False,
           dataCollections=False,
           dataCollectionslong=False,
           goMakeEff=False,
           goMakeEffPhoton=False,
           goMakeBeff=False,
           goMakeSigEff=False,
           goMakeTopPlots=False,
           goTTPlots=False):
    print "# ---------- Submitting condor jobs for {0} ---------- #".format(
        era)
    # split string into list
    if datasets:
        datasets = datasets.split(',')
    else:
        datasets = []
        print "ERROR: datasets not found"
    # sample config files
    eras = [
        "2016", "2017", "2017_BE", "2017_F", "2018", "2018_PreHEM",
        "2018_PostHEM"
    ]
    if era not in eras:
        print "Please use -y to enter era (2016, 2017, 2017_BE, 2017_F, 2018, 2018_PreHEM, or 2018_PostHEM)."
        exit(1)
    # era may include period
    # year does not include period
    year = era[0:4]
    sampleSetsFile = "sampleSets_PostProcessed_" + year + ".cfg"
    sampleCollectionsFile = "sampleCollections_" + year + ".cfg"

    # TopTagger.cfg
    mvaFileName = ""
    with file(environ["CMSSW_BASE"] +
              "/src/ZInvisible/Tools/TopTagger_Tensorflow_2016.cfg"
              ) as meowttcfgFile:
        for line in meowttcfgFile:
            line = line.split("#")[0]
            if "modelFile" in line:
                mvaFileName = line.split("=")[1].strip().strip("\"")
                break

    if verbose:
        print "mvaFileName = {0}".format(mvaFileName)

    #here I hack in the tarball for GMP, this needs to be generalized to the other options

    filestoTransferGMP = [
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/makePlots",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/shapes_njets.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2016_trigger_eff.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2017_trigger_eff.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2018_trigger_eff.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/lepEffHists.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/effhists_GJets.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/njetWgtHists.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/dataMCweights.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/dataMCreweight.root",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/dataMCreweight_allJets.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_bTagEff.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/ISRWeights.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_ISRJets.root",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/PileupHistograms_0121_69p2mb_pm4p6.root",
        environ["CMSSW_BASE"] +
        "/src/TopTagger/TopTagger/test/libTopTagger.so",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagSF_AltTWP.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/tTagEff_2016.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/tTagEff_2017.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/tTagEff_2018.root",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/TopTagger_Tensorflow_2016.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/TopTagger_Tensorflow_2017.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/TopTagger_Tensorflow_2018.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/TopTagger_DiscriminatorFilter_2016.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/TopTagger_DiscriminatorFilter_2017.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/TopTagger_DiscriminatorFilter_2018.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleSets_PostProcessed_2016.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleSets_PostProcessed_2017.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleSets_PostProcessed_2018.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleCollections_2016.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleCollections_2017.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleCollections_2018.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/dc_BkgPred_BinMaps_master.json",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/systematics.json"
    ]

    if mvaFileName:
        filestoTransferGMP += [
            environ["CMSSW_BASE"] +
            "/src/ZInvisible/Tools/%(trainingFile)s" % {
                "trainingFile": mvaFileName
            }
        ]

    filestoTransferGMEP = [
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/calcEffPhoton",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2016_trigger_eff.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2017_trigger_eff.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2018_trigger_eff.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_bTagEff.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/ISRWeights.root",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_ISRJets.root",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/PileupHistograms_0121_69p2mb_pm4p6.root",
        environ["CMSSW_BASE"] +
        "/src/TopTagger/TopTagger/test/libTopTagger.so",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleSets_PostProcessed_2016.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleSets_PostProcessed_2017.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleSets_PostProcessed_2018.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleCollections_2016.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleCollections_2017.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleCollections_2018.cfg"
    ]

    if mvaFileName:
        filestoTransferGMEP += [
            environ["CMSSW_BASE"] +
            "/src/ZInvisible/Tools/%(trainingFile)s" % {
                "trainingFile": mvaFileName
            }
        ]
    if sampleSetsFile:
        filestoTransferGMEP += [
            environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/" + sampleSetsFile
        ]
    if sampleCollectionsFile:
        filestoTransferGMEP += [
            environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/" +
            sampleCollectionsFile
        ]

    #go make plots!
    # old version for submitting from condor directory
    #submitFileGMP = """universe = vanilla
    #Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh
    #Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
    #Should_Transfer_Files = YES
    #WhenToTransferOutput = ON_EXIT
    #Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/gmp.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz
    #Output = logs/makePlots_$(Process).stdout
    #Error = logs/makePlots_$(Process).stderr
    #Log = logs/makePlots_$(Process).log
    #notify_user = ${LOGNAME}@FNAL.GOV
    #x509userproxy = $ENV(X509_USER_PROXY)
    #+maxWallTime = 2880
    #
    #"""
    # new version for submitting from specific directory for submission (use relative instead of full paths for tar files)
    # note that tabs and spaces will appear in condor_submit.txt if you have them here
    submitFileGMP = """universe = vanilla
Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh
Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
Should_Transfer_Files = YES
WhenToTransferOutput = ON_EXIT
Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh,gmp.tar.gz,$ENV(CMSSW_VERSION).tar.gz
Output = logs/makePlots_$(Process).stdout
Error = logs/makePlots_$(Process).stderr
Log = logs/makePlots_$(Process).log
notify_user = ${LOGNAME}@FNAL.GOV
x509userproxy = $ENV(X509_USER_PROXY)
+maxWallTime = 2880

"""

    #Here is the configuration for the Data/MC validation of the TopTagger
    filestoTransferTT = [
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/simpleAnalyzer",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root",
        environ["CMSSW_BASE"] +
        "/src/SusyAnaTools/Tools/data/allINone_bTagEff.root",
        environ["CMSSW_BASE"] +
        "/src/SusyAnaTools/Tools/ISR_Root_Files/ISRWeights.root",
        environ["CMSSW_BASE"] +
        "/src/SusyAnaTools/Tools/ISR_Root_Files/allINone_ISRJets.root",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv",
        environ["CMSSW_BASE"] +
        "/src/SusyAnaTools/Tools/data/PileupHistograms_0121_69p2mb_pm4p6.root",
        environ["CMSSW_BASE"] +
        "/src/TopTagger/TopTagger/test/libTopTagger.so",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleSets_PostProcessed_2016.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleSets_PostProcessed_2017.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleSets_PostProcessed_2018.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleCollections_2016.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleCollections_2017.cfg",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/sampleCollections_2018.cfg"
    ]

    if mvaFileName:
        filestoTransferTT += [
            environ["CMSSW_BASE"] +
            "/src/ZInvisible/Tools/%(trainingFile)s" % {
                "trainingFile": mvaFileName
            }
        ]

    #go make TTopTagger plots!
    submitFileTT = """universe = vanilla
    Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goTTplots.sh
    Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
    Should_Transfer_Files = YES
    WhenToTransferOutput = ON_EXIT
    Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goTTplots.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/TT.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz
    Output = logs/TT_$(Process).stdout
    Error = logs/TT_$(Process).stderr
    Log = logs/TT_$(Process).log
    notify_user = ${LOGNAME}@FNAL.GOV
    x509userproxy = $ENV(X509_USER_PROXY)
    +maxWallTime = 2880
    
    """

    #go make top plots!
    filestoTransferGTP = [
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/makeTopPlots",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root",
        environ["CMSSW_BASE"] +
        "/src/SusyAnaTools/Tools/data/allINone_bTagEff.root",
        environ["CMSSW_BASE"] +
        "/src/SusyAnaTools/Tools/ISR_Root_Files/ISRWeights.root",
        environ["CMSSW_BASE"] +
        "/src/SusyAnaTools/Tools/ISR_Root_Files/allINone_ISRJets.root",
        environ["CMSSW_BASE"] +
        "/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv",
        environ["CMSSW_BASE"] +
        "/src/SusyAnaTools/Tools/data/PileupHistograms_0121_69p2mb_pm4p6.root",
        environ["CMSSW_BASE"] +
        "/src/TopTagger/TopTagger/test/libTopTagger.so",
        environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger.cfg"
    ]

    if mvaFileName:
        filestoTransferTT += [
            environ["CMSSW_BASE"] +
            "/src/ZInvisible/Tools/%(trainingFile)s" % {
                "trainingFile": mvaFileName
            }
        ]

    submitFileGTP = """universe = vanilla
    Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeTopPlots.sh
    Should_Transfer_Files = YES
    WhenToTransferOutput = ON_EXIT
    Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/gtp.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz 
    Output = logs/makePlots_$(Process).stdout
    Error = logs/makePlots_$(Process).stderr
    Log = logs/makePlots_$(Process).log
    notify_user = ${LOGNAME}@FNAL.GOV
    x509userproxy = $ENV(X509_USER_PROXY)
    +maxWallTime = 2880
    
    """

    #go make lepton efficiency
    submitFileGME = """universe = vanilla
    Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEff.sh
    Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
    Should_Transfer_Files = YES
    WhenToTransferOutput = ON_EXIT
    Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/calcEff, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEff.sh, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/zRes.root
    notify_user = ${LOGNAME}@FNAL.GOV
    x509userproxy = $ENV(X509_USER_PROXY)
    
    """

    #go make photon efficiency
    submitFileGMEP = """universe = vanilla
    Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEffPhoton.sh
    Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
    Should_Transfer_Files = YES
    WhenToTransferOutput = ON_EXIT
    Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEffPhoton.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/gmep.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz
    Output = logs/calcEffPhoton_$(Process).stdout
    Error = logs/calcEffPhoton_$(Process).stderr
    Log = logs/calcEffPhoton_$(Process).log
    notify_user = ${LOGNAME}@FNAL.GOV
    x509userproxy = $ENV(X509_USER_PROXY)
    +maxWallTime = 2880
    
    """

    #go B Efficiency calc
    submitFileGBE = """universe = vanilla
    Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeBeff.sh
    Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
    Should_Transfer_Files = YES
    WhenToTransferOutput = ON_EXIT
    Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/beffCalc, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeBeff.sh
    Output = logs/makePlots_$(Process).stdout
    Error = logs/makePlots_$(Process).stderr
    Log = logs/makePlots_$(Process).log
    notify_user = ${LOGNAME}@FNAL.GOV
    x509userproxy = $ENV(X509_USER_PROXY)
    
    """

    #go Signal Efficiency
    submitFileGSE = """universe = vanilla
    Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeSigEff.sh
    Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
    Should_Transfer_Files = YES
    WhenToTransferOutput = ON_EXIT
    Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/makeSignalHistograms, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeSigEff.sh
    Output = logs/makePlots_$(Process).stdout
    Error = logs/makePlots_$(Process).stderr
    Log = logs/makePlots_$(Process).log
    notify_user = ${LOGNAME}@FNAL.GOV
    x509userproxy = $ENV(X509_USER_PROXY)
    
    """

    submitFile = ""
    exeName = ""

    # load samples
    sc = SampleCollection("../" + sampleSetsFile,
                          "../" + sampleCollectionsFile)

    # make directory for condor submission
    now = datetime.datetime.now()
    dirName = "DataMC_%s_submission_%s" % (era,
                                           now.strftime("%Y-%m-%d_%H-%M-%S"))
    os.system("mkdir %s" % dirName)
    #os.chdir(dirName)
    # use class to cd into directory and then return to previous directory
    with cd(dirName):

        def makeExeAndFriendsTarrball(filestoTransfer, fname):
            if not dataCollections and not dataCollectionslong:
                #WORLDSWORSESOLUTIONTOAPROBLEM
                system("mkdir -p WORLDSWORSESOLUTIONTOAPROBLEM")
                for fn in filestoTransfer:
                    system("cd WORLDSWORSESOLUTIONTOAPROBLEM; ln -s %s" % fn)

                tarallinputs = "tar czf %s.tar.gz WORLDSWORSESOLUTIONTOAPROBLEM --dereference" % fname
                if verbose:
                    # Use v option if verbose is set
                    tarallinputs = "tar czvf %s.tar.gz WORLDSWORSESOLUTIONTOAPROBLEM --dereference" % fname
                    print "Create tarball {0}.tag.gz".format(fname)
                    print tarallinputs
                system(tarallinputs)
                system("rm -r WORLDSWORSESOLUTIONTOAPROBLEM")

        if not dataCollections and not dataCollectionslong:
            if verbose:
                print "Create tarball ${CMSSW_VERSION}.tar.gz"
            system(
                "tar --exclude-caches-all --exclude-vcs -zcf ${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION} --exclude=src --exclude=tmp"
            )

        # makeExeAndFriendsTarrball() is necessary now to apply WORLDSWORSESOLUTIONTOAPROBLEM

        if goMakeEff:
            exeName = "calcEff"
            submitFile = submitFileGME
        elif goMakeEffPhoton:
            exeName = "calcEffPhoton"
            submitFile = submitFileGMEP
            makeExeAndFriendsTarrball(filestoTransferGMEP, "gmep")
        elif goMakeBeff:
            exeName = "beffCalc"
            submitFile = submitFileGBE
        elif goMakeSigEff:
            exeName = "makeSignalHistograms"
            submitFile = submitFileGSE
        elif goMakeTopPlots:
            exeName = "makeTopPlots"
            submitFile = submitFileGTP
            makeExeAndFriendsTarrball(filestoTransferGTP, "gtp")
        elif goTTPlots:
            exeName = "simpleAnalyzer"
            submitFile = submitFileTT
            makeExeAndFriendsTarrball(filestoTransferTT, "TT")
        else:
            exeName = "makePlots"
            submitFile = submitFileGMP
            makeExeAndFriendsTarrball(filestoTransferGMP, "gmp")

        nFilesPerJob = numfile

        fileParts = [submitFile]

        if dataCollections or dataCollectionslong:
            scl = sc.sampleCollectionList()
            for sampleCollection in scl:
                sl = sc.sampleList(sampleCollection)
                print sampleCollection
                if dataCollectionslong:
                    sys.stdout.write("\t")
                    for sample in sl:
                        sys.stdout.write("%s  " % sample[1])
                    print ""
                    print ""
            exit(1)

        lumis = sc.sampleCollectionLumiList()
        lumi = sc.getFixedLumi()
        if refLumi != None:
            lumi = lumis[refLumi]
            print "Sample for Ref. Lumi: {0}; normalizing to lumi = {1} pb-1".format(
                refLumi, lumi)

        debug = False
        total_files = 0
        total_events = 0
        jobMap = {}
        for ds in datasets:
            sample_files = 0
            sample_events = 0
            ds = ds.strip()
            if verbose:
                print "# --- {0} --- #".format(ds)
                if "Data" in ds:
                    print "Lumi: {0}".format(lumis[ds])

            # s: file, n:name, e:nEvts
            for s, n, e in sc.sampleList(ds):
                if debug:
                    print "s={0}, n={1}, e={2}".format(s, n, e)
                nFiles = sum(1 for line in open(s))
                nJobs = divideRoundUp(nFiles, nFilesPerJob)
                sample_files += nFiles
                sample_events += e
                total_files += nFiles
                total_events += e
                jobMap[n] = {}
                jobMap[n]["nFilesPerJob"] = nFilesPerJob
                jobMap[n]["nFiles"] = nFiles
                jobMap[n]["nJobs"] = nJobs
                if verbose:
                    print "\t{0}: nFiles={1}, nJobs={2}, n_events={3}".format(
                        n, nFiles, nJobs, e)
                try:
                    f = open(s)
                except IOError:
                    fShort = s.split("/")[-1]
                    if (os.path.isfile(fShort)):
                        os.remove(fShort)
                    system(
                        "xrdcp root://cmseos.fnal.gov/$(echo %s | sed 's|/eos/uscms||') ."
                        % s)
                    if verbose:
                        print "fShort = {0}".format(fShort)
                    f = open(fShort)
                if not f == None:
                    count = 0
                    for l in f:
                        if '.root' in l and not 'failed' in l:
                            count = count + 1
                    for startFileNum in xrange(0, count, nFilesPerJob):
                        fileParts.append(
                            "Arguments = %s $ENV(CMSSW_VERSION) %i %i %f %s %s\n"
                            % (n, nFilesPerJob, startFileNum, lumi, s, era))
                        fileParts.append(
                            "Output = logs/$(Cluster)_$(Process)_%s_%s_%i.stdout\n"
                            % (exeName, n, startFileNum))
                        fileParts.append(
                            "Error = logs/$(Cluster)_$(Process)_%s_%s_%i.stderr\n"
                            % (exeName, n, startFileNum))
                        fileParts.append(
                            "Log = logs/$(Cluster)_$(Process)_%s_%s_%i.log\n" %
                            (exeName, n, startFileNum))
                        fileParts.append("Queue\n\n")

                    f.close()
            # number of files and events in sample
            if verbose:
                print "\t{0}: sample_files={1}, sample_events={2}".format(
                    ds, sample_files, sample_events)

        # total number of files and events
        if verbose:
            print "# --- Totals --- #"
            print "total_files={0}, total_events={1}".format(
                total_files, total_events)

        with open("nJobs.json", "w") as outfile:
            json.dump(jobMap, outfile, sort_keys=True, indent=4)

        fout = open("condor_submit.txt", "w")
        fout.write(''.join(fileParts))
        fout.close()

        if not noSubmit:
            system('mkdir -p logs')
            system("echo 'condor_submit condor_submit.txt'")
            system('condor_submit condor_submit.txt')

    print "Condor submission directory: {0}".format(dirName)
    return dirName