def mergeSample(sample_directory, output_file, runmode='condor', rmunmerged=False): # submit a merging job for one sample # input arguments: # - sample_directory: directory containing root files for one sample # (the files are assumed to be one level down) # - output_file: path to the output root file # - runmode: either "condor", "qsub" or "local" # - rmunmerged: remove the unmerged sample directory cmds = [] cmds.append(mergeCommand(sample_directory, output_file)) if rmunmerged: cmds.append('rm -r {}'.format(sample_directory)) if runmode == 'local': for cmd in cmds: os.system(cmd) elif runmode == 'qsub': script_name = 'qjob_mergedTuples.sh' with open(script_name, 'w') as script: initializeJobScript(script) for cmd in cmds: script.write(cmd) submitQsubJob(script_name) elif runmode == 'condor': ct.submitCommandsAsCondorJob('cjob_mergeTuples', cmds, cmssw_version=CMSSW_VERSION)
def mergeSample( sample_directory, output_directory ): script_name = 'mergedTuples.sh' with open( script_name, 'w' ) as script: initializeJobScript( script ) #merge sample script.write( '{}\n'.format( mergeCommand( sample_directory, output_directory ) ) ) #remove unmerged directory script.write( 'rm -r {}\n'.format( sample_directory ) ) submitQsubJob( script_name )
cmds = [] for year in years: for flavour in flavours: for process in processes: # retrieve the correct samplelist samplelist = os.path.join( samplelistdirectory, 'samples_closureTest_' + process + '_' + year + '.txt') # make the command command = './fillMCClosureTest {} {} {} {} {} {} {} {}'.format( isMCFR, use_mt, process, year, flavour, sampledirectory, samplelist, istestrun) cmds.append(command) # loop over commands and submit jobs if (runmode == 'qsub' or runmode == 'local'): for cmd in cmds: script_name = 'qsub_fillMCClosureTest.sh' with open(script_name, 'w') as script: initializeJobScript(script) script.write('cd {}\n'.format(cwd)) script.write(cmd + '\n') if runmode == 'qsub': submitQsubJob(script_name) else: os.system('bash ' + script_name) elif (runmode == 'condor'): ct.submitCommandsAsCondorCluster('cjob_fillMCClosureTest', cmds, docmsenv=True, cmssw_version='CMSSW_10_2_25')
processes = ['TT', 'DY'] sampledirectory = '/pnfs/iihe/cms/store/user/llambrec/trileptonskim/fakerate/' samplelistdirectory = os.path.abspath('sampleLists') # (see also below to set correct sample list name in loop!) # check if executable exists if not os.path.exists('./closureTest_MC'): print('### ERROR ###: executable does not seem to exist...') print(' run make -f makeClosureTest_MC first.') sys.exit() # loop and sumbit jobs cwd = os.getcwd() for year in years: for flavour in flavours: for process in processes: script_name = 'closureTest_MC.sh' samplelist = os.path.join( samplelistdirectory, 'samples_closureTest_' + process + '_' + year + '.txt') with open(script_name, 'w') as script: initializeJobScript(script) script.write('cd {}\n'.format(cwd)) command = './closureTest_MC {} {} {} {} {} {} {}'.format( isMCFR, use_mt, process, year, flavour, sampledirectory, samplelist) script.write(command + '\n') submitQsubJob(script_name) # alternative: run locally #os.system('bash '+script_name)
output_directory = os.path.join( output_directory_base, 'ntuples_skimmed_{}_version_{}'.format(sample_name, version_name)) if not os.path.exists(output_directory): os.makedirs(output_directory) sample_output_directories.append(output_directory) for sample_directory, sub_directory, output_directory in zip( sample_directories, sample_sub_directories, sample_output_directories): # identify locations of files to process for a given sample root_files = list( listFiles(os.path.join(sample_directory, sub_directory), '.root')) # split_files in lists of files_per_job for chunk in listParts(root_files, files_per_job): # make a job script script_name = 'qjob_skimmer.sh' with open(script_name, 'w') as script: initializeJobScript(script) script.write('cd {}\n'.format(current_directory)) for f in chunk: skim_command = './skimmer {} {} {}\n'.format( f, output_directory, skim_condition) script.write(skim_command) # submit job and catch errors submitQsubJob(script_name, wall_time)