def ClusterParameterEstimator_4SLURM(yml=None,
outputdir=None,
task=None,
isTest=False):
config = Configuration()
config.sbatch_partition = 'cp3'
config.sbatch_qos = 'cp3'
config.cmsswDir = os.path.dirname(os.path.abspath(__file__))
config.sbatch_chdir = os.path.join(config.cmsswDir, outputdir)
config.sbatch_time = '0-02:00'
config.sbatch_memPerCPU = '2000'
config.batchScriptsFilename = "slurmSubmission.sh"
#config.environmentType = 'cms'
config.inputSandboxContent = [
"skimProducer.py" if task == "skim" else
("SiStripHitResol.py" if task == "hitresolution" else
("CPEstimator.py"))
]
config.stageoutFiles = ['*.root']
config.stageoutDir = config.sbatch_chdir
config.inputParamsNames = ["inputFiles", "outputFile", "task", "sample"]
analysisCfgs = os.path.join(config.cmsswDir, yml)
config.inputParams = getTasks(task=task,
analysisCfgs=analysisCfgs,
cmsswDir=config.cmsswDir,
stageoutDir=config.stageoutDir,
isTest=isTest)
shutil.copyfile(analysisCfgs, config.stageoutDir + "/analysis.yml")
config.payload = \
"""
echo ${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}
if [[ "$task" == *"skim"* ]]; then
cmsRun skimProducer.py inputFiles=${inputFiles} outputFile=${outputFile}
elif [[ "$task" == "hitresolution" ]]; then
cmsRun SiStripHitResol.py inputFiles=${inputFiles} outputFile=${outputFile}
else
cmsRun CPEstimator.py inputFiles=${inputFiles} outputFile=${outputFile}
fi
"""
submitWorker = SubmitWorker(config,
submit=True,
yes=True,
debug=True,
quiet=True)
submitWorker()
logger.warning(
'Work still in progress for better workflow ...\n'
'To hadd files and produce plots. Please run as follow when the jobs finish running\n'
'python postprocessing.py --workdir {}\n'
'squeue -u user_name : allows you to check your submitted jobs status\n'
.format(outputdir))
def SlurmRunNano(path= None, outputDIR=None):
config = Configuration()
config.sbatch_partition = 'cp3'
config.sbatch_qos = 'cp3'
config.cmsswDir = os.path.dirname(os.path.abspath(__file__))
config.sbatch_chdir = os.path.join(config.cmsswDir, outputDIR)
config.sbatch_time = '0-06:00'
sbatch_memPerCPU = '2000'
#config.environmentType = 'cms'
config.inputSandboxContent = ["gridpackTolheToNanoGen.sh"]
config.stageoutFiles = ['*.root']
config.stageoutDir = config.sbatch_chdir
config.inputParamsNames = ["gridpack_path","NanoGEN"]
config.inputParams = []
for gridpack_path in glob.glob(os.path.join(os.path.dirname(os.path.abspath(__file__)), path, "*_tarball.tar.xz")):
config.inputParams.append([gridpack_path, "%s%s"%(gridpack_path.split('/')[-1].split('_slc7')[0], ".root")])
workDIR=os.path.dirname(os.path.abspath(__file__))
config.payload = \
"""
if [[ "${NanoGEN}" == *"200p00_125p00"* ]]; then
eval suffix="lowmass_"
else
eval suffix=""
fi
if [[ "${NanoGEN}" == *"bbH"* ]]; then
eval fragment="Hadronizer_TuneCP5_13TeV_aMCatNLO_2p_LHE_pythia8_cff.py"
elif [[ "${NanoGEN}" == *"AToZHTo2L2B"* ]]; then
eval fragment="Hadronizer_TuneCP5_13TeV_AToZHTo2L2B_${suffix}pythia8_PSweights_cff.py"
else
eval fragment="Hadronizer_TuneCP5_13TeV_HToZATo2L2B_${suffix}pythia8_PSweights_cff.py"
fi
pwd
echo ${gridpack_path}
echo ${NanoGEN}
echo ${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}
echo ${fragment}
echo "****************************"
cat ${workDIR}/python/${fragment}
echo "****************************"
bash gridpackTolheToNanoGen.sh ${fragment} ${NanoGEN} ${gridpack_path}
"""
submitWorker = SubmitWorker(config, submit=True, yes=True, debug=True, quiet=True)
submitWorker()
list_check = []
if args.check != '':
print(args.check)
for f in glob.glob(args.check + '/*.root'):
filename = f.replace(args.check, '')
list_check.append(filename)
if len(list_check) == 0:
print(
'Wait ... could not find previous root files, is the path correct ?'
)
sys.exit(1)
# Detail jobs #
for i, job in enumerate(jobs):
job_output = "{}_{}.root".format(output, i)
if job_output in list_check:
print(job_output + ' -> Already done')
continue
slurm_config.inputParams.append(
[job[0], job[1], args.input, job_output])
# Submit job!
print("Submitting job...")
submitWorker = SubmitWorker(slurm_config,
submit=True,
yes=True,
debug=True,
quiet=True)
submitWorker()
print("Done")
def submit_on_slurm(name, args, debug=False):
# Check arguments #
GPU = args.find("--GPU") != -1
output = args.find("--output") != -1
config = Configuration()
config.sbatch_partition = parameters.partition
config.sbatch_qos = parameters.QOS
config.sbatch_workdir = parameters.main_path
config.sbatch_time = parameters.time
#config.sbatch_mem = parameters.mem
config.sbatch_additionalOptions = ['-n ' + str(parameters.tasks)]
if GPU:
config.sbatch_additionalOptions += ['--gres gpu:1', '--export=NONE']
config.inputSandboxContent = []
config.useJobArray = True
config.inputParamsNames = []
config.inputParams = []
if output:
config.inputParamsNames += ["--verbose"]
config.inputParams += [[""]]
if not output:
config.inputParamsNames += ['scan', 'task']
config.payload = """ """
if GPU:
config.payload += "export PYTHONPATH=/root6/lib:$PYTHONPATH\n"
config.payload += "module load cp3\n" # needed on gpu to load slurm_utils
config.payload += "module load slurm/slurm_utils\n"
config.payload += "python3 {script} "
if not output:
config.payload += "--scan ${{scan}} --task ${{task}} "
config.payload += args
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
out_dir = parameters.main_path
slurm_config = copy.deepcopy(config)
slurm_working_dir = os.path.join(out_dir, 'slurm', name + '_' + timestamp)
slurm_config.batchScriptsDir = os.path.join(slurm_working_dir, 'scripts')
slurm_config.inputSandboxDir = slurm_config.batchScriptsDir
slurm_config.stageoutDir = os.path.join(slurm_working_dir, 'output')
slurm_config.stageoutLogsDir = os.path.join(slurm_working_dir, 'logs')
slurm_config.stageoutFiles = ["*.csv", "*.zip", "*.png"]
slurm_config.payload = config.payload.format(
script=os.path.join(out_dir, "ZAMachineLearning.py"))
if not output:
for f in glob.glob(
os.path.join(parameters.main_path, 'split', name, '*.pkl')):
task = os.path.basename(f)
slurm_config.inputParams.append([name, task])
# Submit job!
logging.info("Submitting job...")
if not debug:
submitWorker = SubmitWorker(slurm_config,
submit=True,
yes=True,
debug=False,
quiet=False)
submitWorker()
logging.info("Done")
else:
logging.debug(slurm_config.payload)
logging.debug(slurm_config.inputParamsNames)
logging.debug(slurm_config.inputParams)
logging.info('... don\'t worry, jobs not sent')
def submit_on_slurm(name,args,debug=False):
# Check arguments #
GPU = args.find("--GPU") != -1
output = args.find("--output") != -1
config = Configuration()
config.sbatch_partition = parameters.partition
config.sbatch_qos = parameters.QOS
config.sbatch_chdir = parameters.main_path
config.sbatch_time = parameters.time
config.sbatch_additionalOptions = [parameters.additional_options]
config.sbatch_memPerCPU = parameters.mem
if parameters.partition == 'cp3-gpu':
config.sbatch_additionalOptions += ['--export=NONE']
#if parameters.cpus > 1:
# config.sbatch_additionalOptions += ["--cpus-per-gpu={}".format(parameters.cpus)]
#config.sbatch_additionalOptions += ['--mem-per-gpu={}'.format(parameters.mem)]
elif parameters.partition == 'gpu':
config.sbatch_additionalOptions += ['--gres=gpu:TeslaV100:{}'.format(parameters.gpus),'--export=NONE']
#config.sbatch_additionalOptions += ['--mem-per-gpu={}'.format(parameters.mem)]
#if parameters.cpus > 1:
# config.sbatch_additionalOptions += ["--cpus-per-gpu={}".format(parameters.cpus)]
#if parameters.cpus > 1:
# config.sbatch_additionalOptions += ["--cpus-per-gpu={}".format(parameters.cpus)]
# config.sbatch_additionalOptions += ["--cpus-per-task={}".format(parameters.cpus)]
#config.sbatch_additionalOptions += ["-c {}".format(parameters.cpus)]
else:
if parameters.tasks > 1:
config.sbatch_additionalOptions += ["-n={}".format(parameters.tasks)]
if parameters.cpus > 1:
config.sbatch_additionalOptions += ["--cpus-per-task={}".format(parameters.cpus)]
config.inputSandboxContent = []
config.useJobArray = True
config.inputParamsNames = []
config.inputParams = []
if output:
config.inputParamsNames += ["--verbose"]
config.inputParams += [[""]]
if not output:
config.inputParamsNames += ['scan','task']
if parameters.crossvalidation and parameters.split_per_model:
config.inputParamsNames += ['modelId']
config.payload = ""
if parameters.partition == 'cp3-gpu':
config.payload += "export PYTHONPATH=/python3/lib/python3.6/site-packages/:$PYTHONPATH\n" # GPU tf
config.payload += "export PYTHONPATH=/root6/lib:$PYTHONPATH\n" # ROOT
config.payload += "module load cp3\n" # needed on gpu to load slurm_utils
config.payload += "module load python/python36_sl7_gcc73\n"
config.payload += "module load slurm/slurm_utils\n"
if parameters.partition == 'gpu':
config.payload += "module load releases/2019b_test \n"
config.payload += "module load cp3\n" # needed on gpu to load slurm_utils
config.payload += "module load root/6.12.04-sl7_gcc73 \n"
config.payload += "module load root_numpy \n"
config.payload += "module load TensorFlow \n"
config.payload += "module load slurm/slurm_utils\n"
config.payload += "python3 {script} "
if not output:
config.payload += "--scan ${{scan}} --task ${{task}} "
if parameters.crossvalidation and parameters.split_per_model:
config.payload += "--modelId ${{modelId}}"
config.payload += args
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
out_dir = parameters.main_path
slurm_config = copy.deepcopy(config)
slurm_working_dir = os.path.join(out_dir,'slurm',name+'_'+timestamp)
slurm_config.batchScriptsDir = os.path.join(slurm_working_dir, 'scripts')
slurm_config.inputSandboxDir = slurm_config.batchScriptsDir
slurm_config.stageoutDir = os.path.join(slurm_working_dir, 'output')
slurm_config.stageoutLogsDir = os.path.join(slurm_working_dir, 'logs')
slurm_config.stageoutFiles = ["*.csv","*.zip","*.png"]
slurm_config.payload = config.payload.format(script=os.path.join(out_dir,"HHMachineLearning.py"))
if not output:
for f in glob.glob(os.path.join(parameters.main_path,'split',name,'*.pkl')):
task = os.path.basename(f)
if parameters.crossvalidation and parameters.split_per_model:
for N in range(parameters.N_models):
slurm_config.inputParams.append([name,task,N])
else:
slurm_config.inputParams.append([name,task])
# Submit job!
logging.info("Submitting job...")
if not debug:
submitWorker = SubmitWorker(slurm_config, submit=True, yes=True, debug=False, quiet=False)
submitWorker()
logging.info("Done")
else:
logging.info("Number of jobs : %d"%len(slurm_config.inputParams))
logging.info(slurm_config.payload)
logging.info(slurm_config.inputParamsNames)
for inputParam in slurm_config.inputParams:
logging.info(inputParam)
logging.info('... don\'t worry, jobs not sent')