def job_run(job_id, job_state_file=None): # Determine batch scheduler to user from config. cfg = ConfigObj(os.path.join(pce_root, 'bin', 'onramp_pce_config.cfg'), configspec=os.path.join(pce_root, 'src', 'configspecs', 'onramp_pce_config.cfgspec')) cfg.validate(Validator()) scheduler = Scheduler(cfg['cluster']['batch_scheduler']) ret_dir = os.getcwd() with JobState(job_id, job_state_file) as job_state: run_dir = job_state['run_dir'] run_name = job_state['run_name'] os.chdir(run_dir) # Write batch script. with open('script.sh', 'w') as f: f.write(scheduler.get_batch_script(run_name)) # Schedule job. result = scheduler.schedule(run_dir) if result['status_code'] != 0: _logger.error(result['msg']) with JobState(job_id, job_state_file) as job_state: job_state['state'] = 'Schedule failed' job_state['error'] = result['msg'] os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (result['returncode'], result['msg']) with JobState(job_id, job_state_file) as job_state: job_state['state'] = 'Scheduled' job_state['error'] = None job_state['scheduler_job_num'] = result['job_num'] os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (0, 'Job scheduled')
def launch_job(job_id, mod_id, username, run_name, run_params): """Schedule job launch using system batch scheduler as configured in onramp_pce_config.cfg. Args: job_id (int): Unique identifier for job. mod_id (int): Id for OnRamp educational module to run in this job. username (str): Username of user running the job. run_name (str): Human-readable label for this job run. Returns: Tuple with 0th position being error code and 1st position being string indication of status. """ accepted_states = ['Schedule failed', 'Launch failed', 'Preprocess failed'] _logger.debug('PCE.tools.launch_job() called') # Initialize job state. with JobState(job_id) as job_state: if ('state' in job_state.keys() and job_state['state'] not in accepted_states): msg = 'Job launch already initiated' _logger.warn(msg) return (-1, msg) job_state['job_id'] = job_id job_state['mod_id'] = mod_id job_state['username'] = username job_state['run_name'] = run_name job_state['scheduler_job_num'] = None job_state['state'] = 'Setting up launch' job_state['error'] = None job_state['mod_status_output'] = None job_state['output'] = None job_state['visible_files'] = None job_state['mod_name'] = None job_state['_marked_for_del'] = False _logger.debug('Waiting on ModState at: %s' % time.time()) with ModState(mod_id) as mod_state: _logger.debug('Done waiting on ModState at: %s' % time.time()) if ('state' not in mod_state.keys() or mod_state['state'] != 'Module ready'): msg = 'Module not ready' job_state['state'] = 'Launch failed' job_state['error'] = msg _logger.warn(msg) _logger.warn('mod_state: %s' % str(mod_state)) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (-1, 'Module not ready') job_state['mod_name'] = mod_state['mod_name'] proj_loc = mod_state['installed_path'] mod_name = mod_state['mod_name'] _logger.debug('Testing project location') if not os.path.isdir(proj_loc): msg = 'Project location does not exist' _logger.error(msg) return (-1, msg) _logger.debug('Project location good') # Initialize dir structure. user_dir = os.path.join(os.path.join(pce_root, 'users'), username) user_mod_dir = os.path.join(user_dir, '%s_%d' % (mod_name, mod_id)) run_dir = os.path.join(user_mod_dir, run_name) try: os.mkdir(user_dir) except OSError: # Thrown if dir already exists. pass try: os.mkdir(user_mod_dir) except OSError: # Thrown if dir already exists. pass # The way the following is setup, if a run_dir has already been setup with # this run_name, it will be used (that is, not overwritten) for this launch. try: shutil.copytree(proj_loc, run_dir) except shutil.Error as e: pass if run_params: _logger.debug('Handling run_params') spec = os.path.join(run_dir, 'config/onramp_uioptions.cfgspec') params = ConfigObj(run_params, configspec=spec) result = params.validate(Validator()) if result: with open(os.path.join(run_dir, 'onramp_runparams.cfg'), 'w') as f: params.write(f) else: msg = 'Runparams failed validation' _logger.warn(msg) return (-1, msg) ret_dir = os.getcwd() os.chdir(run_dir) # Preprocess. _logger.info('Calling bin/onramp_preprocess.py') with JobState(job_id) as job_state: job_state['state'] = 'Preprocessing' job_state['error'] = None try: result = check_output([os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_preprocess.py'], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 result = e.output msg = ('Preprocess exited with return status %d and output: %s' % (code, result)) with JobState(job_id) as job_state: job_state['state'] = 'Preprocess failed' job_state['error'] = msg _logger.error(msg) os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (-1, msg) finally: module_log(run_dir, 'preprocess', result) # Determine batch scheduler to user from config. cfg = ConfigObj(os.path.join(pce_root, 'bin', 'onramp_pce_config.cfg'), configspec=os.path.join(pce_root, 'src', 'configspecs', 'onramp_pce_config.cfgspec')) cfg.validate(Validator()) scheduler = Scheduler(cfg['cluster']['batch_scheduler']) # Write batch script. with open('script.sh', 'w') as f: f.write(scheduler.get_batch_script(run_name)) # Schedule job. result = scheduler.schedule(run_dir) if result['status_code'] != 0: _logger.error(result['msg']) with JobState(job_id) as job_state: job_state['state'] = 'Schedule failed' job_state['error'] = result['msg'] os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (result['returncode'], result['msg']) with JobState(job_id) as job_state: job_state['state'] = 'Scheduled' job_state['error'] = None job_state['scheduler_job_num'] = result['job_num'] os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (0, 'Job scheduled')
def job_run(job_id, job_state_file=None): # Determine batch scheduler to user from config. cfg = ConfigObj(os.path.join(pce_root, 'bin', 'onramp_pce_config.cfg'), configspec=os.path.join(pce_root, 'src', 'configspecs', 'onramp_pce_config.cfgspec')) cfg.validate(Validator()) scheduler = Scheduler(cfg['cluster']['batch_scheduler']) _logger.debug("in job_run: trying to launch using scheduler %s", cfg['cluster']['batch_scheduler']) #ret_dir = os.getcwd() with JobState(job_id, job_state_file) as job_state: run_dir = job_state['run_dir'] run_name = job_state['run_name'] os.chdir(run_dir) #_logger.debug("in job_run: attempting to be in %s, really in %s", run_dir, os.get_cwd()) # Load run params: run_np = None run_nodes = None run_cfg = ConfigObj('onramp_runparams.cfg') if 'onramp' in run_cfg.keys(): if 'np' in run_cfg['onramp']: run_np = run_cfg['onramp']['np'] if 'nodes' in run_cfg['onramp']: run_nodes = run_cfg['onramp']['nodes'] _logger.debug("in job_run: loaded params np: %d and nodes: %d", run_np, run_nodes) # Write batch script. with open('script.sh', 'w') as f: if run_np and run_nodes: f.write(scheduler.get_batch_script(run_name, numtasks=run_np, num_nodes=run_nodes)) elif run_np: f.write(scheduler.get_batch_script(run_name, numtasks=run_np)) elif run_nodes: f.write(scheduler.get_batch_script(run_name, num_nodes=run_nodes)) else: f.write(scheduler.get_batch_script(run_name)) # Schedule job. result = scheduler.schedule(run_dir) if result['status_code'] != 0: _logger.error(result['msg']) with JobState(job_id, job_state_file) as job_state: job_state['state'] = 'Schedule failed' job_state['error'] = result['msg'] os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (result['returncode'], result['msg']) with JobState(job_id, job_state_file) as job_state: job_state['state'] = 'Scheduled' job_state['error'] = None job_state['scheduler_job_num'] = result['job_num'] os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (0, 'Job scheduled')