def mergeSample(sample_directory,
output_file,
runmode='condor',
rmunmerged=False):
# submit a merging job for one sample
# input arguments:
# - sample_directory: directory containing root files for one sample
# (the files are assumed to be one level down)
# - output_file: path to the output root file
# - runmode: either "condor", "qsub" or "local"
# - rmunmerged: remove the unmerged sample directory
cmds = []
cmds.append(mergeCommand(sample_directory, output_file))
if rmunmerged: cmds.append('rm -r {}'.format(sample_directory))
if runmode == 'local':
for cmd in cmds:
os.system(cmd)
elif runmode == 'qsub':
script_name = 'qjob_mergedTuples.sh'
with open(script_name, 'w') as script:
initializeJobScript(script)
for cmd in cmds:
script.write(cmd)
submitQsubJob(script_name)
elif runmode == 'condor':
ct.submitCommandsAsCondorJob('cjob_mergeTuples',
cmds,
cmssw_version=CMSSW_VERSION)
def mergeSample( sample_directory, output_directory ):
script_name = 'mergedTuples.sh'
with open( script_name, 'w' ) as script:
initializeJobScript( script )
#merge sample
script.write( '{}\n'.format( mergeCommand( sample_directory, output_directory ) ) )
#remove unmerged directory
script.write( 'rm -r {}\n'.format( sample_directory ) )
submitQsubJob( script_name )
cmds = []
for year in years:
for flavour in flavours:
for process in processes:
# retrieve the correct samplelist
samplelist = os.path.join(
samplelistdirectory,
'samples_closureTest_' + process + '_' + year + '.txt')
# make the command
command = './fillMCClosureTest {} {} {} {} {} {} {} {}'.format(
isMCFR, use_mt, process, year, flavour, sampledirectory,
samplelist, istestrun)
cmds.append(command)
# loop over commands and submit jobs
if (runmode == 'qsub' or runmode == 'local'):
for cmd in cmds:
script_name = 'qsub_fillMCClosureTest.sh'
with open(script_name, 'w') as script:
initializeJobScript(script)
script.write('cd {}\n'.format(cwd))
script.write(cmd + '\n')
if runmode == 'qsub': submitQsubJob(script_name)
else: os.system('bash ' + script_name)
elif (runmode == 'condor'):
ct.submitCommandsAsCondorCluster('cjob_fillMCClosureTest',
cmds,
docmsenv=True,
cmssw_version='CMSSW_10_2_25')
processes = ['TT', 'DY']
sampledirectory = '/pnfs/iihe/cms/store/user/llambrec/trileptonskim/fakerate/'
samplelistdirectory = os.path.abspath('sampleLists')
# (see also below to set correct sample list name in loop!)
# check if executable exists
if not os.path.exists('./closureTest_MC'):
print('### ERROR ###: executable does not seem to exist...')
print(' run make -f makeClosureTest_MC first.')
sys.exit()
# loop and sumbit jobs
cwd = os.getcwd()
for year in years:
for flavour in flavours:
for process in processes:
script_name = 'closureTest_MC.sh'
samplelist = os.path.join(
samplelistdirectory,
'samples_closureTest_' + process + '_' + year + '.txt')
with open(script_name, 'w') as script:
initializeJobScript(script)
script.write('cd {}\n'.format(cwd))
command = './closureTest_MC {} {} {} {} {} {} {}'.format(
isMCFR, use_mt, process, year, flavour, sampledirectory,
samplelist)
script.write(command + '\n')
submitQsubJob(script_name)
# alternative: run locally
#os.system('bash '+script_name)
output_directory = os.path.join(
output_directory_base,
'ntuples_skimmed_{}_version_{}'.format(sample_name, version_name))
if not os.path.exists(output_directory):
os.makedirs(output_directory)
sample_output_directories.append(output_directory)
for sample_directory, sub_directory, output_directory in zip(
sample_directories, sample_sub_directories,
sample_output_directories):
# identify locations of files to process for a given sample
root_files = list(
listFiles(os.path.join(sample_directory, sub_directory), '.root'))
# split_files in lists of files_per_job
for chunk in listParts(root_files, files_per_job):
# make a job script
script_name = 'qjob_skimmer.sh'
with open(script_name, 'w') as script:
initializeJobScript(script)
script.write('cd {}\n'.format(current_directory))
for f in chunk:
skim_command = './skimmer {} {} {}\n'.format(
f, output_directory, skim_condition)
script.write(skim_command)
# submit job and catch errors
submitQsubJob(script_name, wall_time)
