def ClusterParameterEstimator_4SLURM(yml=None, outputdir=None, task=None, isTest=False): config = Configuration() config.sbatch_partition = 'cp3' config.sbatch_qos = 'cp3' config.cmsswDir = os.path.dirname(os.path.abspath(__file__)) config.sbatch_chdir = os.path.join(config.cmsswDir, outputdir) config.sbatch_time = '0-02:00' config.sbatch_memPerCPU = '2000' config.batchScriptsFilename = "slurmSubmission.sh" #config.environmentType = 'cms' config.inputSandboxContent = [ "skimProducer.py" if task == "skim" else ("SiStripHitResol.py" if task == "hitresolution" else ("CPEstimator.py")) ] config.stageoutFiles = ['*.root'] config.stageoutDir = config.sbatch_chdir config.inputParamsNames = ["inputFiles", "outputFile", "task", "sample"] analysisCfgs = os.path.join(config.cmsswDir, yml) config.inputParams = getTasks(task=task, analysisCfgs=analysisCfgs, cmsswDir=config.cmsswDir, stageoutDir=config.stageoutDir, isTest=isTest) shutil.copyfile(analysisCfgs, config.stageoutDir + "/analysis.yml") config.payload = \ """ echo ${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID} if [[ "$task" == *"skim"* ]]; then cmsRun skimProducer.py inputFiles=${inputFiles} outputFile=${outputFile} elif [[ "$task" == "hitresolution" ]]; then cmsRun SiStripHitResol.py inputFiles=${inputFiles} outputFile=${outputFile} else cmsRun CPEstimator.py inputFiles=${inputFiles} outputFile=${outputFile} fi """ submitWorker = SubmitWorker(config, submit=True, yes=True, debug=True, quiet=True) submitWorker() logger.warning( 'Work still in progress for better workflow ...\n' 'To hadd files and produce plots. Please run as follow when the jobs finish running\n' 'python postprocessing.py --workdir {}\n' 'squeue -u user_name : allows you to check your submitted jobs status\n' .format(outputdir))
def SlurmRunNano(path= None, outputDIR=None): config = Configuration() config.sbatch_partition = 'cp3' config.sbatch_qos = 'cp3' config.cmsswDir = os.path.dirname(os.path.abspath(__file__)) config.sbatch_chdir = os.path.join(config.cmsswDir, outputDIR) config.sbatch_time = '0-06:00' sbatch_memPerCPU = '2000' #config.environmentType = 'cms' config.inputSandboxContent = ["gridpackTolheToNanoGen.sh"] config.stageoutFiles = ['*.root'] config.stageoutDir = config.sbatch_chdir config.inputParamsNames = ["gridpack_path","NanoGEN"] config.inputParams = [] for gridpack_path in glob.glob(os.path.join(os.path.dirname(os.path.abspath(__file__)), path, "*_tarball.tar.xz")): config.inputParams.append([gridpack_path, "%s%s"%(gridpack_path.split('/')[-1].split('_slc7')[0], ".root")]) workDIR=os.path.dirname(os.path.abspath(__file__)) config.payload = \ """ if [[ "${NanoGEN}" == *"200p00_125p00"* ]]; then eval suffix="lowmass_" else eval suffix="" fi if [[ "${NanoGEN}" == *"bbH"* ]]; then eval fragment="Hadronizer_TuneCP5_13TeV_aMCatNLO_2p_LHE_pythia8_cff.py" elif [[ "${NanoGEN}" == *"AToZHTo2L2B"* ]]; then eval fragment="Hadronizer_TuneCP5_13TeV_AToZHTo2L2B_${suffix}pythia8_PSweights_cff.py" else eval fragment="Hadronizer_TuneCP5_13TeV_HToZATo2L2B_${suffix}pythia8_PSweights_cff.py" fi pwd echo ${gridpack_path} echo ${NanoGEN} echo ${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID} echo ${fragment} echo "****************************" cat ${workDIR}/python/${fragment} echo "****************************" bash gridpackTolheToNanoGen.sh ${fragment} ${NanoGEN} ${gridpack_path} """ submitWorker = SubmitWorker(config, submit=True, yes=True, debug=True, quiet=True) submitWorker()
required=False, default='') args = parser.parse_args() config = Configuration() config.sbatch_partition = 'cp3' config.sbatch_qos = 'cp3' #config.sbatch_workdir = '.' config.sbatch_time = '0-03:00:00' #config.sbatch_mem = '2048' #config.sbatch_additionalOptions = [] config.inputSandboxContent = [] #['confs/*'] config.useJobArray = True config.inputParamsNames = ['from', 'to', 'input', 'output'] config.inputParams = [] config.payload = """ {executable_path} --from ${{from}} --to ${{to}} --input ${{input}} --output ${{output}} """ #--confs-dir "../confs/" datasets = { 'TTbar': args.max #, #'TW': 20000, #'TbarW': 20000 # 'ttH': 65000, # 'tt': 65000,
def submit_on_slurm(name, args, debug=False): # Check arguments # GPU = args.find("--GPU") != -1 output = args.find("--output") != -1 config = Configuration() config.sbatch_partition = parameters.partition config.sbatch_qos = parameters.QOS config.sbatch_workdir = parameters.main_path config.sbatch_time = parameters.time #config.sbatch_mem = parameters.mem config.sbatch_additionalOptions = ['-n ' + str(parameters.tasks)] if GPU: config.sbatch_additionalOptions += ['--gres gpu:1', '--export=NONE'] config.inputSandboxContent = [] config.useJobArray = True config.inputParamsNames = [] config.inputParams = [] if output: config.inputParamsNames += ["--verbose"] config.inputParams += [[""]] if not output: config.inputParamsNames += ['scan', 'task'] config.payload = """ """ if GPU: config.payload += "export PYTHONPATH=/root6/lib:$PYTHONPATH\n" config.payload += "module load cp3\n" # needed on gpu to load slurm_utils config.payload += "module load slurm/slurm_utils\n" config.payload += "python3 {script} " if not output: config.payload += "--scan ${{scan}} --task ${{task}} " config.payload += args timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') out_dir = parameters.main_path slurm_config = copy.deepcopy(config) slurm_working_dir = os.path.join(out_dir, 'slurm', name + '_' + timestamp) slurm_config.batchScriptsDir = os.path.join(slurm_working_dir, 'scripts') slurm_config.inputSandboxDir = slurm_config.batchScriptsDir slurm_config.stageoutDir = os.path.join(slurm_working_dir, 'output') slurm_config.stageoutLogsDir = os.path.join(slurm_working_dir, 'logs') slurm_config.stageoutFiles = ["*.csv", "*.zip", "*.png"] slurm_config.payload = config.payload.format( script=os.path.join(out_dir, "ZAMachineLearning.py")) if not output: for f in glob.glob( os.path.join(parameters.main_path, 'split', name, '*.pkl')): task = os.path.basename(f) slurm_config.inputParams.append([name, task]) # Submit job! logging.info("Submitting job...") if not debug: submitWorker = SubmitWorker(slurm_config, submit=True, yes=True, debug=False, quiet=False) submitWorker() logging.info("Done") else: logging.debug(slurm_config.payload) logging.debug(slurm_config.inputParamsNames) logging.debug(slurm_config.inputParams) logging.info('... don\'t worry, jobs not sent')
def submit_on_slurm(name,args,debug=False): # Check arguments # GPU = args.find("--GPU") != -1 output = args.find("--output") != -1 config = Configuration() config.sbatch_partition = parameters.partition config.sbatch_qos = parameters.QOS config.sbatch_chdir = parameters.main_path config.sbatch_time = parameters.time config.sbatch_additionalOptions = [parameters.additional_options] config.sbatch_memPerCPU = parameters.mem if parameters.partition == 'cp3-gpu': config.sbatch_additionalOptions += ['--export=NONE'] #if parameters.cpus > 1: # config.sbatch_additionalOptions += ["--cpus-per-gpu={}".format(parameters.cpus)] #config.sbatch_additionalOptions += ['--mem-per-gpu={}'.format(parameters.mem)] elif parameters.partition == 'gpu': config.sbatch_additionalOptions += ['--gres=gpu:TeslaV100:{}'.format(parameters.gpus),'--export=NONE'] #config.sbatch_additionalOptions += ['--mem-per-gpu={}'.format(parameters.mem)] #if parameters.cpus > 1: # config.sbatch_additionalOptions += ["--cpus-per-gpu={}".format(parameters.cpus)] #if parameters.cpus > 1: # config.sbatch_additionalOptions += ["--cpus-per-gpu={}".format(parameters.cpus)] # config.sbatch_additionalOptions += ["--cpus-per-task={}".format(parameters.cpus)] #config.sbatch_additionalOptions += ["-c {}".format(parameters.cpus)] else: if parameters.tasks > 1: config.sbatch_additionalOptions += ["-n={}".format(parameters.tasks)] if parameters.cpus > 1: config.sbatch_additionalOptions += ["--cpus-per-task={}".format(parameters.cpus)] config.inputSandboxContent = [] config.useJobArray = True config.inputParamsNames = [] config.inputParams = [] if output: config.inputParamsNames += ["--verbose"] config.inputParams += [[""]] if not output: config.inputParamsNames += ['scan','task'] if parameters.crossvalidation and parameters.split_per_model: config.inputParamsNames += ['modelId'] config.payload = "" if parameters.partition == 'cp3-gpu': config.payload += "export PYTHONPATH=/python3/lib/python3.6/site-packages/:$PYTHONPATH\n" # GPU tf config.payload += "export PYTHONPATH=/root6/lib:$PYTHONPATH\n" # ROOT config.payload += "module load cp3\n" # needed on gpu to load slurm_utils config.payload += "module load python/python36_sl7_gcc73\n" config.payload += "module load slurm/slurm_utils\n" if parameters.partition == 'gpu': config.payload += "module load releases/2019b_test \n" config.payload += "module load cp3\n" # needed on gpu to load slurm_utils config.payload += "module load root/6.12.04-sl7_gcc73 \n" config.payload += "module load root_numpy \n" config.payload += "module load TensorFlow \n" config.payload += "module load slurm/slurm_utils\n" config.payload += "python3 {script} " if not output: config.payload += "--scan ${{scan}} --task ${{task}} " if parameters.crossvalidation and parameters.split_per_model: config.payload += "--modelId ${{modelId}}" config.payload += args timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') out_dir = parameters.main_path slurm_config = copy.deepcopy(config) slurm_working_dir = os.path.join(out_dir,'slurm',name+'_'+timestamp) slurm_config.batchScriptsDir = os.path.join(slurm_working_dir, 'scripts') slurm_config.inputSandboxDir = slurm_config.batchScriptsDir slurm_config.stageoutDir = os.path.join(slurm_working_dir, 'output') slurm_config.stageoutLogsDir = os.path.join(slurm_working_dir, 'logs') slurm_config.stageoutFiles = ["*.csv","*.zip","*.png"] slurm_config.payload = config.payload.format(script=os.path.join(out_dir,"HHMachineLearning.py")) if not output: for f in glob.glob(os.path.join(parameters.main_path,'split',name,'*.pkl')): task = os.path.basename(f) if parameters.crossvalidation and parameters.split_per_model: for N in range(parameters.N_models): slurm_config.inputParams.append([name,task,N]) else: slurm_config.inputParams.append([name,task]) # Submit job! logging.info("Submitting job...") if not debug: submitWorker = SubmitWorker(slurm_config, submit=True, yes=True, debug=False, quiet=False) submitWorker() logging.info("Done") else: logging.info("Number of jobs : %d"%len(slurm_config.inputParams)) logging.info(slurm_config.payload) logging.info(slurm_config.inputParamsNames) for inputParam in slurm_config.inputParams: logging.info(inputParam) logging.info('... don\'t worry, jobs not sent')
# Job input parameters will be passed to the job via bash environment variables. # There will be one variable per input parameter (e.g. if each of your jobs needs # N input parameters, there will be N bash environment variables defined in each job). # You will have to provide the variable names in the configuration parameter 'inputParamsNames'. # The same variable names will be used for all jobs, while their values will be set per job. # Then it is the user responsability to correctly use these variables in his/ber payload. # The payload is a set of bash lines (which can execute programs in any other language of course). # So one way to retrieve the input variables is to do it in the bash lines of the payload. # The way to access a variable in bash is with the following syntax: ${variable-name}. # This configuration parameter should be a list containing the names of the job input parameters. # It is important that the names in this list and the values in the 'inputParams' list # follow the same order (the slurm_submit tool will assume that this is the case). # Defaults to an empty list (i.e. no input parameters). config.inputParamsNames = ["fileId"] # This configuration parameter should be a list where each element is itself a list containing # the values of the job input parameters. # It is important that the values in this list and the names in the 'inputParamsNames' list # follow the same order (the slurm_submit tool will assume that this is the case). # Defaults to an empty list (i.e. no input parameters). config.inputParams = [[i] for i in range(nnnJobs)] # The payload is code that you want the job to run. It consists of a set of bash lines. # In principle, it can be as simple as "./my_code" (remember to put the file 'my_code' in the input sandbox) # and inside 'my_code' you do whatever you want. # For CMS jobs, the payload can be for example "cmsRun my_code". # (Remember: for CMS jobs, the `cmsenv` command does not need to be included in the payload; # if 'cmsswDir' is specified, `cmsenv` will be executed before running the payload inside that directory). # No validation of the content of this parameter is done.