'file' : inputFileName, 'permissions' : 'mrw-r--r--' # "ordinary" file access permissions } #print "inputFileInfo = %s" % inputFileInfo inputFileInfos.append(inputFileInfo) retVal_make_harvest_scripts = make_harvest_scripts( plot_regex, skim_regex, channel = "ZllRecoilCorrection", sampleToAnalyze = sampleToAnalyze, job_id = version, input_files_info = inputFileInfos, harvester_command = executable_hadd, abort_on_rfcp_error = False, castor_output_directory = outputFileMachine_and_Path, script_directory = configFilePath, merge_script_name = \ os.path.join(configFilePath, "_".join(['submit', sampleToAnalyze, 'merge']) + '.sh'), local_copy_mapper = local_copy_mapper, chunk_size = 2.e+9, # 2 GB run_merging = False, check_old_files = False, max_bsub_concurrent_file_access = 250, verbosity = 0 ) bsubFileNames_harvesting[sampleToAnalyze] = retVal_make_harvest_scripts bsubJobName = "harvest%s" % sampleToAnalyze bsubJobNames_harvesting[sampleToAnalyze] = bsubJobName
yield file def local_copy_mapper(sample): # Define where we want to copy the final output locally return os.path.join( outputFilePath, recoSampleDefinitionsTauIdCommissioning_7TeV['ROOT_FILE_NAMES'][evtSel].replace(".root" + "_harvested.root")) retVal_make_harvest_scripts = make_harvest_scripts( plot_regex, skim_regex, sampleToAnalyze = sample, job_id = "_".join([evtSel, version]), input_source = matches_either(castor_source(inputFilePath)), castor_output_directory = outputFilePath, script_directory = logFilePath, merge_script_name = os.path.join("lxbatch", "_".join(['submit', sample, evtSel, version, 'merge']) + '.sh'), local_copy_mapper = local_copy_mapper, chunk_size = 2.e+9, # 2 GB run_harvesting = False, check_old_files = False, max_bsub_concurrent_file_access = 500, verbosity = 0 ) harvest_scripts.append(retVal_make_harvest_scripts['merge_script_name']) # create "master" shell script shFileName_master = os.path.join("lxbatch", "submit_lxbatch_harvestCommissioningPATTuples_all_%s.sh" % version) shFile_master = open(shFileName_master, "w") for harvest_script in harvest_scripts: shFile_master.write("source %s\n" % harvest_script)
skim_matcher = re.compile(skim_regex) for file in files: #print " unmatched file: %s" % file['path'] if len(SAMPLES_TO_ANALYZE) > 0: isFound = False for sample in SAMPLES_TO_ANALYZE: if file["file"].find(sample) != -1: isFound = True if not isFound: continue if plot_matcher.match(file['file']) or skim_matcher.match(file['file']): #print "--> matched file: %s" % file['path'] yield file def local_copy_mapper(sample): " Define where we want to copy the final output locally " return os.path.join( harvestingFilePath, "_".join(['harvested', channel, sample, jobId]) + ".root") make_harvest_scripts( plot_regex, skim_regex, channel = channel, #input_source = clean_by_crab_id(matches_either(castor_source(analysisFilePath))), input_source = matches_either(castor_source(analysisFilePath)), castor_output_directory = tmpFilePath, local_copy_mapper = local_copy_mapper, chunk_size = 2.e+9, # 2 GB )
" Define where we want to copy the final output locally " return os.path.join( #'/tmp/friis/Run33SYS', #'/tmp/friis/Run32', harvestingFilePath, #'/tmp/friis/%s' % jobId, "_".join(['harvested', channel, sample, jobId]) + ".root") #crab_dirs = glob.glob('crab/crabdir_*_%s' % jobId) crab_dirs = [ 'crab/crabdir_runAHtoMuTau_%s_%s' % (sample, jobId) for sample in [ #'data_Mu_Run2010A_Nov4ReReco', #'data_Mu_Run2010B_Nov4ReReco', #'PPmuXptGt20Mu15', 'Zmumu_powheg' ] ] make_harvest_scripts( plot_regex, skim_regex, channel, #analysisFilePath, castor_source(analysisFilePath), #clean_by_crab_id(castor_source(analysisFilePath)), #crabdir_sources(*crab_dirs), tmpFilePath, local_copy_mapper = local_copy_mapper )
'permissions': 'mrw-r--r--' # "ordinary" file access permissions } #print "inputFileInfo = %s" % inputFileInfo inputFileInfos.append(inputFileInfo) retVal_make_harvest_scripts = make_harvest_scripts( plot_regex, skim_regex, channel = "ZllRecoilCorrection", sampleToAnalyze = sampleToAnalyze, job_id = version, input_files_info = inputFileInfos, harvester_command = executable_hadd, abort_on_rfcp_error = False, castor_output_directory = outputFileMachine_and_Path, script_directory = configFilePath, merge_script_name = \ os.path.join(configFilePath, "_".join(['submit', sampleToAnalyze, 'merge']) + '.sh'), local_copy_mapper = local_copy_mapper, chunk_size = 2.e+9, # 2 GB run_merging = False, check_old_files = False, max_bsub_concurrent_file_access = 250, verbosity = 0 ) bsubFileNames_harvesting[sampleToAnalyze] = retVal_make_harvest_scripts bsubJobName = "harvest%s" % sampleToAnalyze bsubJobNames_harvesting[sampleToAnalyze] = bsubJobName
def local_copy_mapper(sample): " Define where we want to copy the final output locally " return os.path.join( #'/tmp/friis/Run33SYS', #'/tmp/friis/Run32', harvestingFilePath, #'/tmp/friis/%s' % jobId, "_".join(['harvested', channel, sample, jobId]) + ".root") #crab_dirs = glob.glob('crab/crabdir_*_%s' % jobId) crab_dirs = [ 'crab/crabdir_runAHtoMuTau_%s_%s' % (sample, jobId) for sample in [ #'data_Mu_Run2010A_Nov4ReReco', #'data_Mu_Run2010B_Nov4ReReco', #'PPmuXptGt20Mu15', 'Zmumu_powheg' ] ] make_harvest_scripts( plot_regex, skim_regex, channel, #analysisFilePath, castor_source(analysisFilePath), #clean_by_crab_id(castor_source(analysisFilePath)), #crabdir_sources(*crab_dirs), tmpFilePath, local_copy_mapper=local_copy_mapper)
if len(SAMPLES_TO_ANALYZE) > 0: isFound = False for sample in SAMPLES_TO_ANALYZE: if file["file"].find(sample) != -1: isFound = True if not isFound: continue if plot_matcher.match(file['file']) or skim_matcher.match( file['file']): #print "--> matched file: %s" % file['path'] yield file def local_copy_mapper(sample): " Define where we want to copy the final output locally " return os.path.join( harvestingFilePath, "_".join(['harvested', channel, sample, jobId]) + ".root") make_harvest_scripts( plot_regex, skim_regex, channel=channel, #input_source = clean_by_crab_id(matches_either(castor_source(analysisFilePath))), input_source=matches_either(castor_source(analysisFilePath)), castor_output_directory=tmpFilePath, local_copy_mapper=local_copy_mapper, chunk_size=2.e+9, # 2 GB )
'permissions': 'mrw-r--r--' # "ordinary" file access permissions } #print "inputFileInfo = %s" % inputFileInfo inputFileInfos.append(inputFileInfo) retVal_make_harvest_scripts = make_harvest_scripts( plot_regex, skim_regex, channel = channelToAnalyze, sampleToAnalyze = sampleToAnalyze, job_id = "_".join([ metResolution_label, version ]), input_files_info = inputFileInfos, harvester_command = executable_hadd, abort_on_rfcp_error = False, castor_output_directory = harvestingFilePath, script_directory = configFilePath_channel, merge_script_name = \ os.path.join(configFilePath_channel, "_".join([ 'submit', sampleToAnalyze, channelToAnalyze, metResolution_label, 'merge' ]) + '.sh'), local_copy_mapper = local_copy_mapper, chunk_size = 2.e+9, # 2 GB check_old_files = False, max_bsub_concurrent_file_access = 250, verbosity = 0 ) bsubFileNames_harvesting[sampleToAnalyze][channelToAnalyze][ metResolution_label] = retVal_make_harvest_scripts bsubJobName = "harvest%s%s%s" % (sampleToAnalyze, channelToAnalyze, metResolution_label)
def matches_either(files): # Check if the file matches either of the regexes we are interested in. # We do this to skip extra files in the directories before we pass them to # clean_by_crab_id plot_matcher = re.compile(plot_regex) skim_matcher = re.compile(skim_regex) for file in files: if plot_matcher.match(file['file']) or skim_matcher.match( file['file']): yield file def local_copy_mapper(sample): " Define where we want to copy the final output locally " return os.path.join( harvestingFilePath, "_".join(['harvested', channel, sample, jobId]) + ".root") make_harvest_scripts( plot_regex, skim_regex, channel, clean_by_crab_id(matches_either(castor_source(analysisFilePath))), tmpFilePath, local_copy_mapper=local_copy_mapper, chunk_size=2.e+9, # 3 GB )
print tmpFilePath plot_regex = r"plots_%s_(?P<sample>\w+?)_%s_(?P<gridJob>\d*)(_(?P<gridTry>\d*))*_(?P<hash>[a-zA-Z0-9]*).root" % (channel, jobId) skim_regex = r"final_events_%s_(?P<sample>\w+?)_%s_(?P<gridJob>\d*)(_(?P<gridTry>\d*))*_(?P<hash>[a-zA-Z0-9]*).root" % (channel, jobId) def matches_either(files): # Check if the file matches either of the regexes we are interested in. # We do this to skip extra files in the directories before we pass them to # clean_by_crab_id plot_matcher = re.compile(plot_regex) skim_matcher = re.compile(skim_regex) for file in files: if plot_matcher.match(file['file']) or skim_matcher.match(file['file']): yield file def local_copy_mapper(sample): " Define where we want to copy the final output locally " return os.path.join( harvestingFilePath, "_".join(['harvested', channel, sample, jobId]) + ".root") make_harvest_scripts( plot_regex, skim_regex, channel, clean_by_crab_id(matches_either(castor_source(analysisFilePath))), tmpFilePath, local_copy_mapper = local_copy_mapper, chunk_size = 2.e+9, # 3 GB )
'file' : inputFileName, 'permissions' : 'mrw-r--r--' # "ordinary" file access permissions } #print "inputFileInfo = %s" % inputFileInfo inputFileInfos.append(inputFileInfo) retVal_make_harvest_scripts = make_harvest_scripts( plot_regex, skim_regex, channel = channelToAnalyze, sampleToAnalyze = sampleToAnalyze, job_id = "_".join([ metResolution_label, version ]), input_files_info = inputFileInfos, harvester_command = executable_hadd, abort_on_rfcp_error = False, castor_output_directory = harvestingFilePath, script_directory = configFilePath_channel, merge_script_name = \ os.path.join(configFilePath_channel, "_".join([ 'submit', sampleToAnalyze, channelToAnalyze, metResolution_label, 'merge' ]) + '.sh'), local_copy_mapper = local_copy_mapper, chunk_size = 2.e+9, # 2 GB check_old_files = False, max_bsub_concurrent_file_access = 250, verbosity = 0 ) bsubFileNames_harvesting[sampleToAnalyze][channelToAnalyze][metResolution_label] = retVal_make_harvest_scripts bsubJobName = "harvest%s%s%s" % (sampleToAnalyze, channelToAnalyze, metResolution_label) bsubJobNames_harvesting[sampleToAnalyze][channelToAnalyze][metResolution_label] = bsubJobName
def local_copy_mapper(sample): # Define where we want to copy the final output locally return os.path.join( outputFilePath, recoSampleDefinitionsTauIdCommissioning_7TeV['ROOT_FILE_NAMES'] [evtSel].replace(".root" + "_harvested.root")) retVal_make_harvest_scripts = make_harvest_scripts( plot_regex, skim_regex, sampleToAnalyze=sample, job_id="_".join([evtSel, version]), input_source=matches_either(castor_source(inputFilePath)), castor_output_directory=outputFilePath, script_directory=logFilePath, merge_script_name=os.path.join( "lxbatch", "_".join(['submit', sample, evtSel, version, 'merge']) + '.sh'), local_copy_mapper=local_copy_mapper, chunk_size=2.e+9, # 2 GB run_harvesting=False, check_old_files=False, max_bsub_concurrent_file_access=500, verbosity=0) harvest_scripts.append( retVal_make_harvest_scripts['merge_script_name']) # create "master" shell script shFileName_master = os.path.join( "lxbatch",