def _get_module_status_output(run_dir): """Run bin/onramp_status.py for job and return any output. Args: run_dir (str): run dir (as given by job state) for the module. Returns: String containint output to stdout and stderr frob job's bin/onramp_status.py script. """ ret_dir = os.getcwd() # Run bin/onramp_status.py and grab output. os.chdir(run_dir) _logger.debug('Calling bin/onramp_status.py') try: output = check_output([ os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_status.py' ], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 output = ('Status exited with return status %d and output: %s' % (code, e.output)) module_log(run_dir, 'status', output) os.chdir(ret_dir) return output
def _get_module_status_output(run_dir): """Run bin/onramp_status.py for job and return any output. Args: run_dir (str): run dir (as given by job state) for the module. Returns: String containint output to stdout and stderr frob job's bin/onramp_status.py script. """ ret_dir = os.getcwd() # Run bin/onramp_status.py and grab output. os.chdir(run_dir) _logger.debug('Calling bin/onramp_status.py') try: output = check_output([os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_status.py'], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 output = ('Status exited with return status %d and output: %s' % (code, e.output)) module_log(run_dir, 'status', output) os.chdir(ret_dir) return output
def job_postprocess(job_id, job_state_file=None): """Run bin/onramp_postprocess.py for job_id and update state to reflect. Args: job_id (int): Id of the job to launch bin/onramp_postprocess.py for. Returns: Tuple with 0th position being error code and 1st position being string indication of status. """ _logger.info('PCE.tools.jobs._job_postprocess() called') # Get attrs needed. with JobState(job_id, job_state_file) as job_state: username = job_state['username'] mod_id = job_state['mod_id'] run_name = job_state['run_name'] mod_name = job_state['mod_name'] run_dir = job_state['run_dir'] args = (username, mod_name, mod_id, run_name) ret_dir = os.getcwd() os.chdir(run_dir) _logger.debug('Calling bin/onramp_postprocess.py') try: result = check_output([ os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_postprocess.py' ], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 result = e.output msg = ('Postprocess exited with return status %d and output: %s' % (code, result)) with JobState(job_id, job_state_file) as job_state: job_state['state'] = 'Postprocess failed' job_state['error'] = msg _logger.error(msg) os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (-1, msg) finally: module_log(run_dir, 'postprocess', result) # Grab job output. with open('output.txt', 'r') as f: output = f.read() os.chdir(ret_dir) # Update state. with JobState(job_id, job_state_file) as job_state: job_state['state'] = 'Done' job_state['error'] = None job_state['output'] = output if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id)
def job_preprocess(job_id, job_state_file=None): ret_dir = os.getcwd() _logger.info('Calling bin/onramp_preprocess.py') with JobState(job_id, job_state_file) as job_state: job_state['state'] = 'Preprocessing' job_state['error'] = None run_dir = job_state['run_dir'] os.chdir(run_dir) try: result = check_output([ os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_preprocess.py' ], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 result = e.output msg = ('Preprocess exited with return status %d and output: %s' % (code, result)) with JobState(job_id, job_state_file) as job_state: job_state['state'] = 'Preprocess failed' job_state['error'] = msg _logger.error(msg) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (-1, msg) finally: module_log(run_dir, 'preprocess', result) os.chdir(ret_dir) return (0, 'Job preprocess complete')
def _job_postprocess(job_id): """Run bin/onramp_postprocess.py for job_id and update state to reflect. Args: job_id (int): Id of the job to launch bin/onramp_postprocess.py for. Returns: Tuple with 0th position being error code and 1st position being string indication of status. """ _logger.info('PCE.tools.jobs._job_postprocess() called') # Get attrs needed. with JobState(job_id) as job_state: username = job_state['username'] mod_id = job_state['mod_id'] run_name = job_state['run_name'] mod_name = job_state['mod_name'] args = (username, mod_name, mod_id, run_name) run_dir = os.path.join(pce_root, 'users/%s/%s_%d/%s' % args) ret_dir = os.getcwd() os.chdir(run_dir) _logger.debug('Calling bin/onramp_postprocess.py') try: result = check_output([os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_postprocess.py'], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 result = e.output msg = ('Postprocess exited with return status %d and output: %s' % (code, result)) with JobState(job_id) as job_state: job_state['state'] = 'Postprocess failed' job_state['error'] = msg _logger.error(msg) os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (-1, msg) finally: module_log(run_dir, 'postprocess', result) # Grab job output. with open('output.txt', 'r') as f: output = f.read() os.chdir(ret_dir) # Update state. with JobState(job_id) as job_state: job_state['state'] = 'Done' job_state['error'] = None job_state['output'] = output if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id)
def _get_module_status_output(job_id, job_state_file=None): """Run bin/onramp_status.py for job and return any output. Args: job_id (int): Id of the job to launch bin/onramp_status.py for. Returns: String containint output to stdout and stderr frob job's bin/onramp_status.py script. """ # Get attrs needed. with JobState(job_id, job_state_file) as job_state: username = job_state['username'] mod_id = job_state['mod_id'] run_name = job_state['run_name'] mod_name = job_state['mod_name'] run_dir = job_state['run_dir'] #args = (username, mod_name, mod_id, run_name) #run_dir = os.path.join(pce_root, 'users/%s/%s_%d/%s' % args) ret_dir = os.getcwd() # Run bin/onramp_status.py and grab output. os.chdir(run_dir) _logger.debug('Calling bin/onramp_status.py') try: output = check_output([ os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_status.py' ], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 output = ('Status exited with return status %d and output: %s' % (code, e.output)) module_log(run_dir, 'status', output) os.chdir(ret_dir) return output
def job_preprocess(job_id, job_state_file=None): ret_dir = os.getcwd() _logger.info('Calling bin/onramp_preprocess.py') _logger.debug('Want JobState (preprocess) at: %s' % time.time()) with JobState(job_id, job_state_file) as job_state: _logger.debug('In JobState (preprocess) at: %s' % time.time()) _logger.debug('preprocess PID: %d' % os.getpid()) job_state['state'] = 'Preprocessing' job_state['error'] = None run_dir = job_state['run_dir'] os.chdir(run_dir) _logger.debug('Done with JobState (preprocess) at: %s' % time.time()) try: result = check_output([os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_preprocess.py'], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 result = e.output msg = ('Preprocess exited with return status %d and output: %s' % (code, result)) with JobState(job_id, job_state_file) as job_state: job_state['state'] = 'Preprocess failed' job_state['error'] = msg _logger.error(msg) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (-1, msg) finally: module_log(run_dir, 'preprocess', result) os.chdir(ret_dir) return (0, 'Job preprocess complete')
def _get_module_status_output(job_id): """Run bin/onramp_status.py for job and return any output. Args: job_id (int): Id of the job to launch bin/onramp_status.py for. Returns: String containint output to stdout and stderr frob job's bin/onramp_status.py script. """ # Get attrs needed. with JobState(job_id) as job_state: username = job_state['username'] mod_id = job_state['mod_id'] run_name = job_state['run_name'] mod_name = job_state['mod_name'] args = (username, mod_name, mod_id, run_name) run_dir = os.path.join(pce_root, 'users/%s/%s_%d/%s' % args) ret_dir = os.getcwd() # Run bin/onramp_status.py and grab output. os.chdir(run_dir) _logger.debug('Calling bin/onramp_status.py') try: output = check_output([os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_status.py'], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 output = ('Status exited with return status %d and output: %s' % (code, e.output)) module_log(run_dir, 'status', output) os.chdir(ret_dir) return output
def deploy_module(mod_id, verbose=False): """Deploy an installed OnRamp educational module. Args: mod_id (int): Id of the module to be deployed. Kwargs: verbose (bool): Increases status output if True. Returns: Tuple with 0th position being error code and 1st position being string indication of status. """ mod_dir = None not_installed_states = ['Available', 'Checkout in Progress', 'Checkout failed'] with ModState(mod_id) as mod_state: if ('state' not in mod_state.keys() or mod_state['state'] in not_installed_states): return (-1, 'Module %d not installed' % mod_id) if mod_state['state'] == 'Deploy in progress': return (-1, 'Deployment already underway for module %d' % mod_id) mod_state['state'] = 'Deploy in progress' mod_state['error'] = None mod_dir = mod_state['installed_path'] ret_dir = os.getcwd() _logger.debug('ret_dir: %s' % ret_dir) os.chdir(mod_dir) try: _logger.debug('Calling bin/onramp_deploy.py') _logger.debug('CWD: %s' % os.getcwd()) output = check_output([os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_deploy.py'], stderr=STDOUT) _logger.debug('Back from bin/onramp_deploy.py') except CalledProcessError as e: _logger.debug('CalledProcessError from bin/onramp_deploy.py') code = e.returncode if code > 127: code -= 256 output = e.output if code != 1: with ModState(mod_id) as mod_state: msg = ('Deploy exited with return status %d and output: %s' % (code, output)) _logger.debug(msg) mod_state['state'] = 'Deploy failed' mod_state['error'] = msg if mod_state['_marked_for_del']: _delete_module(mod_state) return (-3, 'Module %d deleted' % mod_id) return (-1, msg) with ModState(mod_id) as mod_state: msg = 'Admin required' _logger.debug(msg) mod_state['state'] = msg mod_state['error'] = output if mod_state['_marked_for_del']: _delete_module(mod_state) return (-3, 'Module %d deleted' % mod_id) return (1, msg) except OSError as e1: output = str(e1) _logger.debug('OSError from bin/onramp_deploy.py') _logger.debug(e1) with ModState(mod_id) as mod_state: mod_state['state'] = 'Deploy failed' mod_state['error'] = str(e1) return (-1, str(e1)) finally: os.chdir(ret_dir) module_log(mod_dir, 'deploy', output) _logger.debug("Updating state to 'Module ready'") with ModState(mod_id) as mod_state: mod_state['state'] = 'Module ready' mod_state['error'] = None if mod_state['_marked_for_del']: _delete_module(mod_state) return (-3, 'Module %d deleted' % mod_id) return (0, 'Module %d ready' % mod_id)
def deploy_module(mod_id, verbose=False, mod_state_file=None): """Deploy an installed OnRamp educational module. Args: mod_id (int): Id of the module to be deployed. Kwargs: verbose (bool): Increases status output if True. Returns: Tuple with 0th position being error code and 1st position being string indication of status. """ mod_dir = None not_installed_states = ["Available", "Checkout in Progress", "Checkout failed"] with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: if "state" not in mod_state.keys() or mod_state["state"] in not_installed_states: return (-1, "Module %d not installed" % mod_id) if mod_state["state"] == "Deploy in progress": return (-1, "Deployment already underway for module %d" % mod_id) mod_state["state"] = "Deploy in progress" mod_state["error"] = None mod_dir = mod_state["installed_path"] ret_dir = os.getcwd() _logger.debug("ret_dir: %s" % ret_dir) os.chdir(mod_dir) try: _logger.debug("Calling bin/onramp_deploy.py") _logger.debug("CWD: %s" % os.getcwd()) output = check_output([os.path.join(pce_root, "src/env/bin/python"), "bin/onramp_deploy.py"], stderr=STDOUT) _logger.debug("Back from bin/onramp_deploy.py") except CalledProcessError as e: _logger.debug("CalledProcessError from bin/onramp_deploy.py") code = e.returncode if code > 127: code -= 256 output = e.output if code != 1: with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: msg = "Deploy exited with return status %d and output: %s" % (code, output) _logger.debug(msg) mod_state["state"] = "Deploy failed" mod_state["error"] = msg if mod_state["_marked_for_del"]: _delete_module(mod_state) return (-3, "Module %d deleted" % mod_id) return (-1, msg) with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: msg = "Admin required" _logger.debug(msg) mod_state["state"] = msg mod_state["error"] = output if mod_state["_marked_for_del"]: _delete_module(mod_state) return (-3, "Module %d deleted" % mod_id) return (1, msg) except OSError as e1: output = str(e1) _logger.debug("OSError from bin/onramp_deploy.py") _logger.debug(e1) with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: mod_state["state"] = "Deploy failed" mod_state["error"] = str(e1) return (-1, str(e1)) finally: os.chdir(ret_dir) module_log(mod_dir, "deploy", output) _logger.debug("Updating state to 'Module ready'") with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: mod_state["state"] = "Module ready" mod_state["error"] = None if mod_state["_marked_for_del"]: _delete_module(mod_state) return (-3, "Module %d deleted" % mod_id) return (0, "Module %d ready" % mod_id)
def launch_job(job_id, mod_id, username, run_name, run_params): """Schedule job launch using system batch scheduler as configured in onramp_pce_config.cfg. Args: job_id (int): Unique identifier for job. mod_id (int): Id for OnRamp educational module to run in this job. username (str): Username of user running the job. run_name (str): Human-readable label for this job run. Returns: Tuple with 0th position being error code and 1st position being string indication of status. """ accepted_states = ['Schedule failed', 'Launch failed', 'Preprocess failed'] _logger.debug('PCE.tools.launch_job() called') # Initialize job state. with JobState(job_id) as job_state: if ('state' in job_state.keys() and job_state['state'] not in accepted_states): msg = 'Job launch already initiated' _logger.warn(msg) return (-1, msg) job_state['job_id'] = job_id job_state['mod_id'] = mod_id job_state['username'] = username job_state['run_name'] = run_name job_state['scheduler_job_num'] = None job_state['state'] = 'Setting up launch' job_state['error'] = None job_state['mod_status_output'] = None job_state['output'] = None job_state['visible_files'] = None job_state['mod_name'] = None job_state['_marked_for_del'] = False _logger.debug('Waiting on ModState at: %s' % time.time()) with ModState(mod_id) as mod_state: _logger.debug('Done waiting on ModState at: %s' % time.time()) if ('state' not in mod_state.keys() or mod_state['state'] != 'Module ready'): msg = 'Module not ready' job_state['state'] = 'Launch failed' job_state['error'] = msg _logger.warn(msg) _logger.warn('mod_state: %s' % str(mod_state)) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (-1, 'Module not ready') job_state['mod_name'] = mod_state['mod_name'] proj_loc = mod_state['installed_path'] mod_name = mod_state['mod_name'] _logger.debug('Testing project location') if not os.path.isdir(proj_loc): msg = 'Project location does not exist' _logger.error(msg) return (-1, msg) _logger.debug('Project location good') # Initialize dir structure. user_dir = os.path.join(os.path.join(pce_root, 'users'), username) user_mod_dir = os.path.join(user_dir, '%s_%d' % (mod_name, mod_id)) run_dir = os.path.join(user_mod_dir, run_name) try: os.mkdir(user_dir) except OSError: # Thrown if dir already exists. pass try: os.mkdir(user_mod_dir) except OSError: # Thrown if dir already exists. pass # The way the following is setup, if a run_dir has already been setup with # this run_name, it will be used (that is, not overwritten) for this launch. try: shutil.copytree(proj_loc, run_dir) except shutil.Error as e: pass if run_params: _logger.debug('Handling run_params') spec = os.path.join(run_dir, 'config/onramp_uioptions.cfgspec') params = ConfigObj(run_params, configspec=spec) result = params.validate(Validator()) if result: with open(os.path.join(run_dir, 'onramp_runparams.cfg'), 'w') as f: params.write(f) else: msg = 'Runparams failed validation' _logger.warn(msg) return (-1, msg) ret_dir = os.getcwd() os.chdir(run_dir) # Preprocess. _logger.info('Calling bin/onramp_preprocess.py') with JobState(job_id) as job_state: job_state['state'] = 'Preprocessing' job_state['error'] = None try: result = check_output([os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_preprocess.py'], stderr=STDOUT) except CalledProcessError as e: code = e.returncode if code > 127: code -= 256 result = e.output msg = ('Preprocess exited with return status %d and output: %s' % (code, result)) with JobState(job_id) as job_state: job_state['state'] = 'Preprocess failed' job_state['error'] = msg _logger.error(msg) os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (-1, msg) finally: module_log(run_dir, 'preprocess', result) # Determine batch scheduler to user from config. cfg = ConfigObj(os.path.join(pce_root, 'bin', 'onramp_pce_config.cfg'), configspec=os.path.join(pce_root, 'src', 'configspecs', 'onramp_pce_config.cfgspec')) cfg.validate(Validator()) scheduler = Scheduler(cfg['cluster']['batch_scheduler']) # Write batch script. with open('script.sh', 'w') as f: f.write(scheduler.get_batch_script(run_name)) # Schedule job. result = scheduler.schedule(run_dir) if result['status_code'] != 0: _logger.error(result['msg']) with JobState(job_id) as job_state: job_state['state'] = 'Schedule failed' job_state['error'] = result['msg'] os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (result['returncode'], result['msg']) with JobState(job_id) as job_state: job_state['state'] = 'Scheduled' job_state['error'] = None job_state['scheduler_job_num'] = result['job_num'] os.chdir(ret_dir) if job_state['_marked_for_del']: _delete_job(job_state) return (-2, 'Job %d deleted' % job_id) return (0, 'Job scheduled')
def deploy_module(mod_id, verbose=False, mod_state_file=None): """Deploy an installed OnRamp educational module. Args: mod_id (int): Id of the module to be deployed. Kwargs: verbose (bool): Increases status output if True. Returns: Tuple with 0th position being error code and 1st position being string indication of status. """ mod_dir = None not_installed_states = [ 'Available', 'Checkout in Progress', 'Checkout failed' ] with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: if ('state' not in mod_state.keys() or mod_state['state'] in not_installed_states): return (-1, 'Module %d not installed' % mod_id) if mod_state['state'] == 'Deploy in progress': return (-1, 'Deployment already underway for module %d' % mod_id) mod_state['state'] = 'Deploy in progress' mod_state['error'] = None mod_dir = mod_state['installed_path'] ret_dir = os.getcwd() _logger.debug('ret_dir: %s' % ret_dir) os.chdir(mod_dir) try: _logger.debug('Calling bin/onramp_deploy.py') _logger.debug('CWD: %s' % os.getcwd()) output = check_output([ os.path.join(pce_root, 'src/env/bin/python'), 'bin/onramp_deploy.py' ], stderr=STDOUT) _logger.debug('Back from bin/onramp_deploy.py') except CalledProcessError as e: _logger.debug('CalledProcessError from bin/onramp_deploy.py') code = e.returncode if code > 127: code -= 256 output = e.output if code != 1: with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: msg = ('Deploy exited with return status %d and output: %s' % (code, output)) _logger.debug(msg) mod_state['state'] = 'Deploy failed' mod_state['error'] = msg if mod_state['_marked_for_del']: _delete_module(mod_state) return (-3, 'Module %d deleted' % mod_id) return (-1, msg) with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: msg = 'Admin required' _logger.debug(msg) mod_state['state'] = msg mod_state['error'] = output if mod_state['_marked_for_del']: _delete_module(mod_state) return (-3, 'Module %d deleted' % mod_id) return (1, msg) except OSError as e1: output = str(e1) _logger.debug('OSError from bin/onramp_deploy.py') _logger.debug(e1) with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: mod_state['state'] = 'Deploy failed' mod_state['error'] = str(e1) return (-1, str(e1)) finally: os.chdir(ret_dir) module_log(mod_dir, 'deploy', output) _logger.debug("Updating state to 'Module ready'") with ModState(mod_id, mod_state_file=mod_state_file) as mod_state: mod_state['state'] = 'Module ready' mod_state['error'] = None if mod_state['_marked_for_del']: _delete_module(mod_state) return (-3, 'Module %d deleted' % mod_id) return (0, 'Module %d ready' % mod_id)