def create_hadd_python_file(self, inputFiles, outputFile, hadd_stage_name):
sbatch_hadd_file = os.path.join(
self.outputDir,
"sbatch_hadd_%s_%s.py" % (self.channel, hadd_stage_name))
tools_createScript_sbatch_hadd(sbatch_hadd_file,
inputFiles,
outputFile,
hadd_stage_name,
self.workingDir,
pool_id=self.pool_id)
return sbatch_hadd_file
def addToMakefile_hadd(self, lines_makefile):
"""Add hadd targets to the Makefile
"""
for hadd_out, hadd_in in self.hadd_records.iteritems():
hadd_in_files = hadd_in['output_files']
hadd_fileset_id = hadd_in['fileset_id']
sbatch_hadd_file = os.path.join(
self.outputDir, DKEY_HADD, "sbatch_hadd_%s_%s_%d.py" % (self.channel, 'cat', hadd_fileset_id)
)
tools_createScript_sbatch_hadd(sbatch_hadd_file, hadd_in_files, hadd_out, 'cat', self.workingDir, False)
lines_makefile.append("%s: %s" % (hadd_out, " ".join(hadd_in_files)))
lines_makefile.append("\t%s %s" % ("rm -f", hadd_out))
lines_makefile.append("\t%s %s" % ("python", sbatch_hadd_file))
lines_makefile.append("")
def create_hadd_python_file(self, inputFiles, outputFile, hadd_stage_name):
sbatch_hadd_file = os.path.join(self.dirs[DKEY_SCRIPTS], "sbatch_hadd_%s_%s.py" % (self.channel, hadd_stage_name))
sbatch_hadd_file = sbatch_hadd_file.replace(".root", "")
scriptFile = os.path.join(self.dirs[DKEY_SCRIPTS], os.path.basename(sbatch_hadd_file).replace(".py", ".sh"))
logFile = os.path.join(self.dirs[DKEY_LOGS], os.path.basename(sbatch_hadd_file).replace(".py", ".log"))
sbatch_hadd_dir = os.path.join(self.dirs[DKEY_HADD_RT], self.channel, hadd_stage_name) if self.dirs[DKEY_HADD_RT] else ''
self.num_jobs['hadd'] += tools_createScript_sbatch_hadd(
sbatch_hadd_file, inputFiles, outputFile, scriptFile, logFile, self.workingDir, auxDirName = sbatch_hadd_dir,
pool_id = uuid.uuid4(), verbose = self.verbose,
)
return sbatch_hadd_file
def addToMakefile_hadd(self, lines_makefile):
"""Add hadd targets to the Makefile
"""
for hadd_out, hadd_in in self.hadd_records.iteritems():
hadd_in_files = hadd_in['output_files']
hadd_fileset_id = hadd_in['fileset_id']
process_name = hadd_in['process_name']
sbatch_hadd_file = os.path.join(
self.cfgDir, DKEY_HADD, self.channel, process_name,
"sbatch_hadd_cat_%s_%d.py" % (process_name, hadd_fileset_id))
sbatch_hadd_shFile = os.path.join(
self.cfgDir, DKEY_HADD, self.channel, process_name,
"sbatch_hadd_cat_%s_%d.sh" % (process_name, hadd_fileset_id))
sbatch_hadd_logFile = os.path.join(
self.cfgDir, DKEY_HADD, self.channel, process_name,
"sbatch_hadd_cat_%s_%d.log" % (process_name, hadd_fileset_id))
sbatch_hadd_dir = os.path.join(
self.cfgDir,
DKEY_HADD_RT,
self.channel,
process_name,
)
sbatch_hadd_logFile = get_log_version((sbatch_hadd_logFile, ))
tools_createScript_sbatch_hadd(
sbatch_script_file_name=sbatch_hadd_file,
input_file_names=hadd_in_files,
output_file_name=hadd_out,
script_file_name=sbatch_hadd_shFile,
log_file_name=sbatch_hadd_logFile[0],
working_dir=self.workingDir,
waitForJobs=False,
auxDirName=sbatch_hadd_dir,
pool_id=self.pool_id,
use_home=self.use_home,
)
lines_makefile.append("%s: %s" %
(hadd_out, " ".join(hadd_in_files)))
lines_makefile.append("\t%s %s" % ("rm -f", hadd_out))
lines_makefile.append("\t%s %s" % ("python", sbatch_hadd_file))
lines_makefile.append("")
def create_hadd_python_file(self,
inputFiles,
outputFile,
hadd_stage_name,
process_name=''):
sbatch_hadd_file = os.path.join(self.dirs[DKEY_SCRIPTS],
"sbatch_hadd_%s.py" % hadd_stage_name)
sbatch_hadd_file = sbatch_hadd_file.replace(".root", "")
scriptsDir = self.dirs[process_name][
DKEY_SCRIPTS] if process_name else self.dirs[DKEY_SCRIPTS]
logDir = self.dirs[process_name][
DKEY_LOGS] if process_name else self.dirs[DKEY_LOGS]
haddRtDir = self.dirs[process_name][
DKEY_HADD_RT] if process_name else self.dirs[DKEY_HADD_RT]
scriptFile = os.path.join(
scriptsDir,
os.path.basename(sbatch_hadd_file).replace(".py", ".sh"))
logFile = os.path.join(
logDir,
os.path.basename(sbatch_hadd_file).replace(".py", ".log"))
sbatch_hadd_dir = os.path.join(haddRtDir, hadd_stage_name)
self.num_jobs['hadd'] += tools_createScript_sbatch_hadd(
sbatch_script_file_name=sbatch_hadd_file,
input_file_names=inputFiles,
output_file_name=outputFile,
script_file_name=scriptFile,
log_file_name=logFile,
working_dir=self.workingDir,
auxDirName=sbatch_hadd_dir,
pool_id=uuid.uuid4(),
verbose=self.verbose,
dry_run=self.dry_run,
use_home=self.use_home,
max_input_files_per_job=20,
min_file_size=-1,
)
return sbatch_hadd_file
def create_hadd_python_file(self, inputFiles, outputFile, hadd_stage_name):
sbatch_hadd_file = os.path.join(self.outputDir, "sbatch_hadd_%s_%s.py" % (self.channel, hadd_stage_name))
tools_createScript_sbatch_hadd(sbatch_hadd_file, inputFiles, outputFile, hadd_stage_name, self.workingDir)
return sbatch_hadd_file
