示例#1
0
def ClusterParameterEstimator_4SLURM(yml=None,
                                     outputdir=None,
                                     task=None,
                                     isTest=False):
    config = Configuration()
    config.sbatch_partition = 'cp3'
    config.sbatch_qos = 'cp3'
    config.cmsswDir = os.path.dirname(os.path.abspath(__file__))
    config.sbatch_chdir = os.path.join(config.cmsswDir, outputdir)
    config.sbatch_time = '0-02:00'
    config.sbatch_memPerCPU = '2000'
    config.batchScriptsFilename = "slurmSubmission.sh"
    #config.environmentType = 'cms'
    config.inputSandboxContent = [
        "skimProducer.py" if task == "skim" else
        ("SiStripHitResol.py" if task == "hitresolution" else
         ("CPEstimator.py"))
    ]
    config.stageoutFiles = ['*.root']
    config.stageoutDir = config.sbatch_chdir
    config.inputParamsNames = ["inputFiles", "outputFile", "task", "sample"]

    analysisCfgs = os.path.join(config.cmsswDir, yml)
    config.inputParams = getTasks(task=task,
                                  analysisCfgs=analysisCfgs,
                                  cmsswDir=config.cmsswDir,
                                  stageoutDir=config.stageoutDir,
                                  isTest=isTest)
    shutil.copyfile(analysisCfgs, config.stageoutDir + "/analysis.yml")
    config.payload = \
    """
    echo ${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}
    if [[ "$task" == *"skim"* ]]; then
        cmsRun skimProducer.py inputFiles=${inputFiles} outputFile=${outputFile}
    elif [[ "$task" == "hitresolution" ]]; then
        cmsRun SiStripHitResol.py inputFiles=${inputFiles} outputFile=${outputFile}
    else
        cmsRun CPEstimator.py inputFiles=${inputFiles} outputFile=${outputFile}
    fi
    """

    submitWorker = SubmitWorker(config,
                                submit=True,
                                yes=True,
                                debug=True,
                                quiet=True)
    submitWorker()
    logger.warning(
        'Work still in progress for better workflow ...\n'
        'To hadd files and produce plots. Please run as follow when the jobs finish running\n'
        'python postprocessing.py --workdir {}\n'
        'squeue -u user_name  : allows you to check your submitted jobs status\n'
        .format(outputdir))
示例#2
0
# Do NOT use absolute paths. Keep in mind also that relative paths are not only to locate the files locally,
# but also when the tarball is unpacked the relative paths are respected. That is, if you add in the input sandbox a line "mydir/*.txt",
# then all txt files present in ./mydir will be included in the input sandbox and when unpacked in the worker nodes
# you will have a directory mydir with all these txt files inside.
# Defaults to an empty list (i.e. no input sandbox).
config.inputSandboxContent = [
    "run.py", "*.out", "Dict_rdict.pcm", "config.json", "submit.sh"
]
config.inputSandboxContent = [
    "runVerify.py", "EXEC", "config.json", "submitVerify.sh"
]

config.inputSandboxDir = config.sbatch_workdir
config.inputSandboxFilename = ''
config.batchScriptsDir = config.sbatch_workdir
config.batchScriptsFilename = ''

config.stageout = True
config.stageoutFiles = ["*.root"]
config.stageoutDir = "~/storage/HH_HLLHC/DELPHES_YR/dump"

config.writeLogsOnWN = True
config.separateStdoutStderrLogs = False
config.stdoutFilename = "slurm-${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}.out"
config.stderrFilename = ""

config.stageoutLogs = True
config.stageoutLogsDir = config.stageoutDir + '/logs'

config.useJobArray = True
config.numJobs = None