'file'        : inputFileName,
                'permissions' : 'mrw-r--r--' # "ordinary" file access permissions
            }
            #print "inputFileInfo = %s" % inputFileInfo
            inputFileInfos.append(inputFileInfo)

    retVal_make_harvest_scripts = make_harvest_scripts(
        plot_regex,
        skim_regex,
        channel = "ZllRecoilCorrection",
        sampleToAnalyze = sampleToAnalyze,
        job_id = version,
        input_files_info = inputFileInfos,
        harvester_command = executable_hadd,
        abort_on_rfcp_error = False,
        castor_output_directory = outputFileMachine_and_Path,
        script_directory = configFilePath,
        merge_script_name = \
        os.path.join(configFilePath, "_".join(['submit', sampleToAnalyze, 'merge']) + '.sh'),
        local_copy_mapper = local_copy_mapper,
        chunk_size = 2.e+9, # 2 GB
        run_merging = False,
        check_old_files = False,
        max_bsub_concurrent_file_access = 250,
        verbosity = 0
    )

    bsubFileNames_harvesting[sampleToAnalyze] = retVal_make_harvest_scripts

    bsubJobName = "harvest%s" % sampleToAnalyze
    bsubJobNames_harvesting[sampleToAnalyze] = bsubJobName
                     yield file  

        def local_copy_mapper(sample):
            # Define where we want to copy the final output locally 
            return os.path.join(
                outputFilePath,
                recoSampleDefinitionsTauIdCommissioning_7TeV['ROOT_FILE_NAMES'][evtSel].replace(".root" + "_harvested.root"))

        retVal_make_harvest_scripts = make_harvest_scripts(
            plot_regex,
            skim_regex,
            sampleToAnalyze = sample,
            job_id = "_".join([evtSel, version]),
            input_source = matches_either(castor_source(inputFilePath)),
            castor_output_directory = outputFilePath,
            script_directory = logFilePath,
            merge_script_name = os.path.join("lxbatch", "_".join(['submit', sample, evtSel, version, 'merge']) + '.sh'),
            local_copy_mapper = local_copy_mapper,
            chunk_size = 2.e+9, # 2 GB
            run_harvesting = False,
            check_old_files = False,
            max_bsub_concurrent_file_access = 500,
            verbosity = 0
        )

        harvest_scripts.append(retVal_make_harvest_scripts['merge_script_name'])

# create "master" shell script
shFileName_master = os.path.join("lxbatch", "submit_lxbatch_harvestCommissioningPATTuples_all_%s.sh" % version)
shFile_master = open(shFileName_master, "w")
for harvest_script in harvest_scripts:
    shFile_master.write("source %s\n" % harvest_script)
    skim_matcher = re.compile(skim_regex)
    for file in files:
        #print " unmatched file: %s" % file['path']
        if len(SAMPLES_TO_ANALYZE) > 0:
	  isFound = False
          for sample in SAMPLES_TO_ANALYZE:
              if file["file"].find(sample) != -1:
                  isFound = True
          if not isFound:
	    continue
        if plot_matcher.match(file['file']) or skim_matcher.match(file['file']):
            #print "--> matched file: %s" % file['path']
            yield file  

def local_copy_mapper(sample):
    " Define where we want to copy the final output locally "
    return os.path.join(
        harvestingFilePath,
        "_".join(['harvested', channel, sample, jobId]) + ".root")

make_harvest_scripts(
    plot_regex,
    skim_regex,
    channel = channel,
    #input_source = clean_by_crab_id(matches_either(castor_source(analysisFilePath))),
    input_source = matches_either(castor_source(analysisFilePath)),
    castor_output_directory = tmpFilePath,
    local_copy_mapper = local_copy_mapper,
    chunk_size = 2.e+9, # 2 GB
)
        " Define where we want to copy the final output locally "
        return os.path.join(
            #'/tmp/friis/Run33SYS',
            #'/tmp/friis/Run32',
            harvestingFilePath,
            #'/tmp/friis/%s' % jobId,
            "_".join(['harvested', channel, sample, jobId]) + ".root")

    #crab_dirs = glob.glob('crab/crabdir_*_%s' % jobId)
    crab_dirs = [
        'crab/crabdir_runAHtoMuTau_%s_%s' % (sample, jobId)
        for sample in [
            #'data_Mu_Run2010A_Nov4ReReco',
            #'data_Mu_Run2010B_Nov4ReReco',
            #'PPmuXptGt20Mu15',
            'Zmumu_powheg'
        ]
    ]

    make_harvest_scripts(
        plot_regex,
        skim_regex,
        channel,
        #analysisFilePath,
        castor_source(analysisFilePath),
        #clean_by_crab_id(castor_source(analysisFilePath)),
        #crabdir_sources(*crab_dirs),
        tmpFilePath,
        local_copy_mapper = local_copy_mapper
    )
                'permissions':
                'mrw-r--r--'  # "ordinary" file access permissions
            }
            #print "inputFileInfo = %s" % inputFileInfo
            inputFileInfos.append(inputFileInfo)

    retVal_make_harvest_scripts = make_harvest_scripts(
        plot_regex,
        skim_regex,
        channel = "ZllRecoilCorrection",
        sampleToAnalyze = sampleToAnalyze,
        job_id = version,
        input_files_info = inputFileInfos,
        harvester_command = executable_hadd,
        abort_on_rfcp_error = False,
        castor_output_directory = outputFileMachine_and_Path,
        script_directory = configFilePath,
        merge_script_name = \
        os.path.join(configFilePath, "_".join(['submit', sampleToAnalyze, 'merge']) + '.sh'),
        local_copy_mapper = local_copy_mapper,
        chunk_size = 2.e+9, # 2 GB
        run_merging = False,
        check_old_files = False,
        max_bsub_concurrent_file_access = 250,
        verbosity = 0
    )

    bsubFileNames_harvesting[sampleToAnalyze] = retVal_make_harvest_scripts

    bsubJobName = "harvest%s" % sampleToAnalyze
    bsubJobNames_harvesting[sampleToAnalyze] = bsubJobName
Exemple #6
0
    def local_copy_mapper(sample):
        " Define where we want to copy the final output locally "
        return os.path.join(
            #'/tmp/friis/Run33SYS',
            #'/tmp/friis/Run32',
            harvestingFilePath,
            #'/tmp/friis/%s' % jobId,
            "_".join(['harvested', channel, sample, jobId]) + ".root")

    #crab_dirs = glob.glob('crab/crabdir_*_%s' % jobId)
    crab_dirs = [
        'crab/crabdir_runAHtoMuTau_%s_%s' % (sample, jobId) for sample in [
            #'data_Mu_Run2010A_Nov4ReReco',
            #'data_Mu_Run2010B_Nov4ReReco',
            #'PPmuXptGt20Mu15',
            'Zmumu_powheg'
        ]
    ]

    make_harvest_scripts(
        plot_regex,
        skim_regex,
        channel,
        #analysisFilePath,
        castor_source(analysisFilePath),
        #clean_by_crab_id(castor_source(analysisFilePath)),
        #crabdir_sources(*crab_dirs),
        tmpFilePath,
        local_copy_mapper=local_copy_mapper)
        if len(SAMPLES_TO_ANALYZE) > 0:
            isFound = False
            for sample in SAMPLES_TO_ANALYZE:
                if file["file"].find(sample) != -1:
                    isFound = True
            if not isFound:
                continue
        if plot_matcher.match(file['file']) or skim_matcher.match(
                file['file']):
            #print "--> matched file: %s" % file['path']
            yield file


def local_copy_mapper(sample):
    " Define where we want to copy the final output locally "
    return os.path.join(
        harvestingFilePath,
        "_".join(['harvested', channel, sample, jobId]) + ".root")


make_harvest_scripts(
    plot_regex,
    skim_regex,
    channel=channel,
    #input_source = clean_by_crab_id(matches_either(castor_source(analysisFilePath))),
    input_source=matches_either(castor_source(analysisFilePath)),
    castor_output_directory=tmpFilePath,
    local_copy_mapper=local_copy_mapper,
    chunk_size=2.e+9,  # 2 GB
)
                    'permissions':
                    'mrw-r--r--'  # "ordinary" file access permissions
                }
                #print "inputFileInfo = %s" % inputFileInfo
                inputFileInfos.append(inputFileInfo)

            retVal_make_harvest_scripts = make_harvest_scripts(
                plot_regex,
                skim_regex,
                channel = channelToAnalyze,
                sampleToAnalyze = sampleToAnalyze,
                job_id = "_".join([ metResolution_label, version ]),
                input_files_info = inputFileInfos,
                harvester_command = executable_hadd,
                abort_on_rfcp_error = False,
                castor_output_directory = harvestingFilePath,
                script_directory = configFilePath_channel,
                merge_script_name = \
                os.path.join(configFilePath_channel, "_".join([ 'submit', sampleToAnalyze, channelToAnalyze,
                                                                metResolution_label, 'merge' ]) + '.sh'),
                local_copy_mapper = local_copy_mapper,
                chunk_size = 2.e+9, # 2 GB
                check_old_files = False,
                max_bsub_concurrent_file_access = 250,
                verbosity = 0
            )

            bsubFileNames_harvesting[sampleToAnalyze][channelToAnalyze][
                metResolution_label] = retVal_make_harvest_scripts

            bsubJobName = "harvest%s%s%s" % (sampleToAnalyze, channelToAnalyze,
                                             metResolution_label)
Exemple #9
0

def matches_either(files):
    # Check if the file matches either of the regexes we are interested in.
    # We do this to skip extra files in the directories before we pass them to
    # clean_by_crab_id
    plot_matcher = re.compile(plot_regex)
    skim_matcher = re.compile(skim_regex)
    for file in files:
        if plot_matcher.match(file['file']) or skim_matcher.match(
                file['file']):
            yield file


def local_copy_mapper(sample):
    " Define where we want to copy the final output locally "
    return os.path.join(
        harvestingFilePath,
        "_".join(['harvested', channel, sample, jobId]) + ".root")


make_harvest_scripts(
    plot_regex,
    skim_regex,
    channel,
    clean_by_crab_id(matches_either(castor_source(analysisFilePath))),
    tmpFilePath,
    local_copy_mapper=local_copy_mapper,
    chunk_size=2.e+9,  # 3 GB
)
print tmpFilePath
    
plot_regex = r"plots_%s_(?P<sample>\w+?)_%s_(?P<gridJob>\d*)(_(?P<gridTry>\d*))*_(?P<hash>[a-zA-Z0-9]*).root" % (channel, jobId)
skim_regex = r"final_events_%s_(?P<sample>\w+?)_%s_(?P<gridJob>\d*)(_(?P<gridTry>\d*))*_(?P<hash>[a-zA-Z0-9]*).root" % (channel, jobId)

def matches_either(files):
    # Check if the file matches either of the regexes we are interested in.
    # We do this to skip extra files in the directories before we pass them to
    # clean_by_crab_id
    plot_matcher = re.compile(plot_regex)
    skim_matcher = re.compile(skim_regex)
    for file in files:
        if plot_matcher.match(file['file']) or skim_matcher.match(file['file']):
            yield file

def local_copy_mapper(sample):
    " Define where we want to copy the final output locally "
    return os.path.join(
        harvestingFilePath,
        "_".join(['harvested', channel, sample, jobId]) + ".root")            

make_harvest_scripts(
    plot_regex,
    skim_regex,
    channel,
    clean_by_crab_id(matches_either(castor_source(analysisFilePath))),
    tmpFilePath,
    local_copy_mapper = local_copy_mapper,
    chunk_size = 2.e+9, # 3 GB
)
                    'file'        : inputFileName,
                    'permissions' : 'mrw-r--r--' # "ordinary" file access permissions
                }
                #print "inputFileInfo = %s" % inputFileInfo
                inputFileInfos.append(inputFileInfo)

            retVal_make_harvest_scripts = make_harvest_scripts(
                plot_regex,
                skim_regex,
                channel = channelToAnalyze,
                sampleToAnalyze = sampleToAnalyze,
                job_id = "_".join([ metResolution_label, version ]),
                input_files_info = inputFileInfos,
                harvester_command = executable_hadd,
                abort_on_rfcp_error = False,
                castor_output_directory = harvestingFilePath,
                script_directory = configFilePath_channel,
                merge_script_name = \
                os.path.join(configFilePath_channel, "_".join([ 'submit', sampleToAnalyze, channelToAnalyze,
                                                                metResolution_label, 'merge' ]) + '.sh'),
                local_copy_mapper = local_copy_mapper,
                chunk_size = 2.e+9, # 2 GB
                check_old_files = False,
                max_bsub_concurrent_file_access = 250,
                verbosity = 0
            )

            bsubFileNames_harvesting[sampleToAnalyze][channelToAnalyze][metResolution_label] = retVal_make_harvest_scripts

            bsubJobName = "harvest%s%s%s" % (sampleToAnalyze, channelToAnalyze, metResolution_label)
            bsubJobNames_harvesting[sampleToAnalyze][channelToAnalyze][metResolution_label] = bsubJobName
Exemple #12
0
        def local_copy_mapper(sample):
            # Define where we want to copy the final output locally
            return os.path.join(
                outputFilePath,
                recoSampleDefinitionsTauIdCommissioning_7TeV['ROOT_FILE_NAMES']
                [evtSel].replace(".root" + "_harvested.root"))

        retVal_make_harvest_scripts = make_harvest_scripts(
            plot_regex,
            skim_regex,
            sampleToAnalyze=sample,
            job_id="_".join([evtSel, version]),
            input_source=matches_either(castor_source(inputFilePath)),
            castor_output_directory=outputFilePath,
            script_directory=logFilePath,
            merge_script_name=os.path.join(
                "lxbatch",
                "_".join(['submit', sample, evtSel, version, 'merge']) +
                '.sh'),
            local_copy_mapper=local_copy_mapper,
            chunk_size=2.e+9,  # 2 GB
            run_harvesting=False,
            check_old_files=False,
            max_bsub_concurrent_file_access=500,
            verbosity=0)

        harvest_scripts.append(
            retVal_make_harvest_scripts['merge_script_name'])

# create "master" shell script
shFileName_master = os.path.join(
    "lxbatch",