def submit_on_slurm(name, args, debug=False):
# Check arguments #
GPU = args.find("--GPU") != -1
output = args.find("--output") != -1
config = Configuration()
config.sbatch_partition = parameters.partition
config.sbatch_qos = parameters.QOS
config.sbatch_workdir = parameters.main_path
config.sbatch_time = parameters.time
#config.sbatch_mem = parameters.mem
config.sbatch_additionalOptions = ['-n ' + str(parameters.tasks)]
if GPU:
config.sbatch_additionalOptions += ['--gres gpu:1', '--export=NONE']
config.inputSandboxContent = []
config.useJobArray = True
config.inputParamsNames = []
config.inputParams = []
if output:
config.inputParamsNames += ["--verbose"]
config.inputParams += [[""]]
if not output:
config.inputParamsNames += ['scan', 'task']
config.payload = """ """
if GPU:
config.payload += "export PYTHONPATH=/root6/lib:$PYTHONPATH\n"
config.payload += "module load cp3\n" # needed on gpu to load slurm_utils
config.payload += "module load slurm/slurm_utils\n"
config.payload += "python3 {script} "
if not output:
config.payload += "--scan ${{scan}} --task ${{task}} "
config.payload += args
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
out_dir = parameters.main_path
slurm_config = copy.deepcopy(config)
slurm_working_dir = os.path.join(out_dir, 'slurm', name + '_' + timestamp)
slurm_config.batchScriptsDir = os.path.join(slurm_working_dir, 'scripts')
slurm_config.inputSandboxDir = slurm_config.batchScriptsDir
slurm_config.stageoutDir = os.path.join(slurm_working_dir, 'output')
slurm_config.stageoutLogsDir = os.path.join(slurm_working_dir, 'logs')
slurm_config.stageoutFiles = ["*.csv", "*.zip", "*.png"]
slurm_config.payload = config.payload.format(
script=os.path.join(out_dir, "ZAMachineLearning.py"))
if not output:
for f in glob.glob(
os.path.join(parameters.main_path, 'split', name, '*.pkl')):
task = os.path.basename(f)
slurm_config.inputParams.append([name, task])
# Submit job!
logging.info("Submitting job...")
if not debug:
submitWorker = SubmitWorker(slurm_config,
submit=True,
yes=True,
debug=False,
quiet=False)
submitWorker()
logging.info("Done")
else:
logging.debug(slurm_config.payload)
logging.debug(slurm_config.inputParamsNames)
logging.debug(slurm_config.inputParams)
logging.info('... don\'t worry, jobs not sent')
nnnJobs = 0
with open("config.json", "r") as f:
jjj = json.loads(f.read())
nnnJobs = len(jjj)
from CP3SlurmUtils.Configuration import Configuration
config = Configuration()
#--------------------------------------------------------------------------------
# 1. SLURM sbatch command options
#--------------------------------------------------------------------------------
config.sbatch_partition = 'cp3'
config.sbatch_qos = 'cp3'
config.sbatch_workdir = './slurm/'
config.sbatch_time = '0-02:30:00'
config.sbatch_mem = '1024'
config.sbatch_output = ''
config.sbatch_error = ''
# Example: config.sbatch_additionalOptions = ['--mail-type=END', '--mail-user=<your-email-address>']
config.sbatch_additionalOptions = []
#--------------------------------------------------------------------------------
# 2. User batch script parameters that are same for all jobs
#--------------------------------------------------------------------------------
config.environmentType = ''