def createScript_sbatch(self):
"""Creates the python script necessary to submit the Ntuple production jobs to the batch system
"""
tools_createScript_sbatch(
sbatch_script_file_name = self.sbatchFile_prodNtuple,
executable = self.executable_prodNtuple,
cfg_file_names = self.cfgFiles_prodNtuple_modified,
input_file_names = self.inputFiles,
output_file_names = self.outputFiles,
log_file_names = self.logFiles_prodNtuple,
working_dir = self.workingDir,
max_num_jobs = self.max_num_jobs,
cvmfs_error_log = self.cvmfs_error_log
)
def createScript_sbatch(self):
"""Creates the python script necessary to submit the analysis jobs to the batch system
"""
tools_createScript_sbatch(
sbatch_script_file_name = self.sbatchFile_analyze,
executable = self.executable_analyze,
cfg_file_names = self.cfgFiles_analyze_modified,
input_file_names = self.ntupleFiles,
output_file_names = self.histogramFiles,
log_file_names = self.logFiles_analyze,
working_dir = self.workingDir,
max_num_jobs = self.max_num_jobs,
cvmfs_error_log = self.cvmfs_error_log
)
def createScript_sbatch(self):
"""Creates the python script necessary to submit the MEM jobs to the batch system
"""
tools_createScript_sbatch(
sbatch_script_file_name = self.sbatchFile_addMEM,
executable = self.executable_addMEM,
cfg_file_names = self.cfgFiles_addMEM_modified,
input_file_names = self.inputFiles,
output_file_names = self.outputFiles,
log_file_names = self.logFiles_addMEM,
working_dir = self.workingDir,
max_num_jobs = self.max_mem_integrations,
cvmfs_error_log = self.cvmfs_error_log,
)
def createScript_sbatch(self):
"""Creates the python script necessary to submit the analysis jobs to the batch system
"""
tools_createScript_sbatch(
sbatch_script_file_name=self.sbatchFile_analyze,
executable=self.executable_analyze,
cfg_file_names=self.cfgFiles_analyze_modified,
input_file_names=self.ntupleFiles,
output_file_names=self.histogramFiles,
log_file_names=self.logFiles_analyze,
working_dir=self.workingDir,
max_num_jobs=self.max_num_jobs,
cvmfs_error_log=self.cvmfs_error_log,
pool_id=self.pool_id,
)
def createScript_sbatch(self):
"""Creates the python script necessary to submit the Ntuple production jobs to the batch system
"""
tools_createScript_sbatch(
sbatch_script_file_name=self.sbatchFile_preselNtuple,
executable=self.executable_preselNtuple,
cfg_file_names=self.cfgFiles_preselNtuple_modified,
input_file_names=self.inputFiles,
output_file_names=self.outputFiles,
log_file_names=self.logFiles_preselNtuple,
working_dir=self.workingDir,
max_num_jobs=self.max_num_jobs,
cvmfs_error_log=self.cvmfs_error_log,
pool_id=self.pool_id,
)
def createScript_sbatch(self):
"""Creates the python script necessary to submit the Ntuple production jobs to the batch system
"""
num_jobs = tools_createScript_sbatch(
sbatch_script_file_name=self.sbatchFile_prodNtuple,
executable=self.executable,
command_line_parameters=self.cfgFiles_prodNtuple_modified,
input_file_names=self.inputFiles,
output_file_names=self.outputFiles,
script_file_names={
key: value.replace(".py", ".sh").replace("_cfg", "")
for key, value in self.cfgFiles_prodNtuple_modified.items()
},
log_file_names=self.logFiles_prodNtuple,
working_dir=self.workingDir,
max_num_jobs=self.max_num_jobs,
cvmfs_error_log=self.cvmfs_error_log,
pool_id=self.pool_id,
verbose=self.verbose,
job_template_file='sbatch-node.produce.sh.template',
dry_run=self.dry_run,
use_home=self.use_home,
validate_outputs=self.check_output_files,
)
return num_jobs
def createScript_sbatch(self):
"""Creates the python script necessary to submit the MEM jobs to the batch system
"""
tools_createScript_sbatch(
sbatch_script_file_name=self.sbatchFile_addMEM,
executable=self.executable_addMEM,
command_line_parameters=self.cfgFiles_addMEM_modified,
input_file_names=self.inputFiles,
output_file_names=self.outputFiles,
script_file_names=self.shFiles_addMEM_modified,
log_file_names=self.logFiles_addMEM,
working_dir=self.workingDir,
max_num_jobs=
100000000, # it's really silly to limit the number of jobs; use an enormous number as the ,,fix''
cvmfs_error_log=self.cvmfs_error_log,
pool_id=self.pool_id,
)
def createScript_sbatch(self):
"""Creates the python script necessary to submit the Ntuple production jobs to the batch system
"""
tools_createScript_sbatch(
sbatch_script_file_name=self.sbatchFile_prodNtuple,
executable=self.executable_prodNtuple,
command_line_parameters=self.cfgFiles_prodNtuple_modified,
input_file_names=self.inputFiles,
output_file_names=self.outputFiles,
script_file_names={
key: value.replace(".py", ".sh").replace("_cfg", "")
for key, value in self.cfgFiles_prodNtuple_modified.items()
},
log_file_names=self.logFiles_prodNtuple,
working_dir=self.workingDir,
max_num_jobs=self.max_num_jobs,
cvmfs_error_log=self.cvmfs_error_log,
pool_id=self.pool_id,
)
def createScript_sbatch(
self,
executable,
sbatchFile,
jobOptions,
key_cfg_file='cfgFile_path',
key_input_file='inputFiles',
key_output_file='outputFile',
key_log_file='logFile',
key_script_file='scriptFile',
):
num_jobs = tools_createScript_sbatch(
sbatch_script_file_name=sbatchFile,
executable=executable,
command_line_parameters={
key: value[key_cfg_file]
for key, value in jobOptions.items()
},
input_file_names={
key: value[key_input_file]
for key, value in jobOptions.items()
},
output_file_names={
key: value[key_output_file]
for key, value in jobOptions.items()
},
script_file_names={
key: value[key_script_file]
for key, value in jobOptions.items()
},
log_file_names={
key: value[key_log_file]
for key, value in jobOptions.items()
},
keep_logs=self.keep_logs,
working_dir=self.workingDir,
max_num_jobs=self.max_num_jobs,
cvmfs_error_log=self.cvmfs_error_log,
pool_id=uuid.uuid4(),
verbose=self.verbose,
dry_run=self.dry_run,
job_template_file='sbatch-node.produce.sh.template',
validate_outputs=self.check_output_files,
min_file_size=-1,
use_home=self.use_home,
)
return num_jobs
def createScript_sbatch(self, executable, sbatchFile, jobOptions,
key_cfg_file = 'cfgFile_modified', key_input_file = 'inputFile', key_output_file = 'outputFile', key_log_file = 'logFile'):
"""Creates the python script necessary to submit 'generic' (addBackgrounds, addBackgroundFakes/addBackgroundFlips) jobs to the batch system
"""
num_jobs = tools_createScript_sbatch(
sbatch_script_file_name = sbatchFile,
executable = executable,
command_line_parameters = { key: value[key_cfg_file] for key, value in jobOptions.items() },
input_file_names = { key: value[key_input_file] for key, value in jobOptions.items() },
output_file_names = { key: value[key_output_file] for key, value in jobOptions.items() },
script_file_names = { key: value[key_cfg_file].replace(".py", ".sh").replace("_cfg", "") for key, value in jobOptions.items() },
log_file_names = { key: value[key_log_file] for key, value in jobOptions.items() },
working_dir = self.workingDir,
max_num_jobs = self.max_num_jobs,
cvmfs_error_log = self.cvmfs_error_log,
pool_id = uuid.uuid4(),
verbose = self.verbose,
)
return num_jobs