def _get_module_status_output(run_dir):
"""Run bin/onramp_status.py for job and return any output.
Args:
run_dir (str): run dir (as given by job state) for the module.
Returns:
String containint output to stdout and stderr frob job's
bin/onramp_status.py script.
"""
ret_dir = os.getcwd()
# Run bin/onramp_status.py and grab output.
os.chdir(run_dir)
_logger.debug('Calling bin/onramp_status.py')
try:
output = check_output([
os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_status.py'
],
stderr=STDOUT)
except CalledProcessError as e:
code = e.returncode
if code > 127:
code -= 256
output = ('Status exited with return status %d and output: %s' %
(code, e.output))
module_log(run_dir, 'status', output)
os.chdir(ret_dir)
return output
def _get_module_status_output(run_dir):
"""Run bin/onramp_status.py for job and return any output.
Args:
run_dir (str): run dir (as given by job state) for the module.
Returns:
String containint output to stdout and stderr frob job's
bin/onramp_status.py script.
"""
ret_dir = os.getcwd()
# Run bin/onramp_status.py and grab output.
os.chdir(run_dir)
_logger.debug('Calling bin/onramp_status.py')
try:
output = check_output([os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_status.py'], stderr=STDOUT)
except CalledProcessError as e:
code = e.returncode
if code > 127:
code -= 256
output = ('Status exited with return status %d and output: %s'
% (code, e.output))
module_log(run_dir, 'status', output)
os.chdir(ret_dir)
return output
def job_postprocess(job_id, job_state_file=None):
"""Run bin/onramp_postprocess.py for job_id and update state to reflect.
Args:
job_id (int): Id of the job to launch bin/onramp_postprocess.py for.
Returns:
Tuple with 0th position being error code and 1st position being string
indication of status.
"""
_logger.info('PCE.tools.jobs._job_postprocess() called')
# Get attrs needed.
with JobState(job_id, job_state_file) as job_state:
username = job_state['username']
mod_id = job_state['mod_id']
run_name = job_state['run_name']
mod_name = job_state['mod_name']
run_dir = job_state['run_dir']
args = (username, mod_name, mod_id, run_name)
ret_dir = os.getcwd()
os.chdir(run_dir)
_logger.debug('Calling bin/onramp_postprocess.py')
try:
result = check_output([
os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_postprocess.py'
],
stderr=STDOUT)
except CalledProcessError as e:
code = e.returncode
if code > 127:
code -= 256
result = e.output
msg = ('Postprocess exited with return status %d and output: %s' %
(code, result))
with JobState(job_id, job_state_file) as job_state:
job_state['state'] = 'Postprocess failed'
job_state['error'] = msg
_logger.error(msg)
os.chdir(ret_dir)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (-1, msg)
finally:
module_log(run_dir, 'postprocess', result)
# Grab job output.
with open('output.txt', 'r') as f:
output = f.read()
os.chdir(ret_dir)
# Update state.
with JobState(job_id, job_state_file) as job_state:
job_state['state'] = 'Done'
job_state['error'] = None
job_state['output'] = output
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
def job_preprocess(job_id, job_state_file=None):
ret_dir = os.getcwd()
_logger.info('Calling bin/onramp_preprocess.py')
with JobState(job_id, job_state_file) as job_state:
job_state['state'] = 'Preprocessing'
job_state['error'] = None
run_dir = job_state['run_dir']
os.chdir(run_dir)
try:
result = check_output([
os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_preprocess.py'
],
stderr=STDOUT)
except CalledProcessError as e:
code = e.returncode
if code > 127:
code -= 256
result = e.output
msg = ('Preprocess exited with return status %d and output: %s' %
(code, result))
with JobState(job_id, job_state_file) as job_state:
job_state['state'] = 'Preprocess failed'
job_state['error'] = msg
_logger.error(msg)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (-1, msg)
finally:
module_log(run_dir, 'preprocess', result)
os.chdir(ret_dir)
return (0, 'Job preprocess complete')
def _job_postprocess(job_id):
"""Run bin/onramp_postprocess.py for job_id and update state to reflect.
Args:
job_id (int): Id of the job to launch bin/onramp_postprocess.py for.
Returns:
Tuple with 0th position being error code and 1st position being string
indication of status.
"""
_logger.info('PCE.tools.jobs._job_postprocess() called')
# Get attrs needed.
with JobState(job_id) as job_state:
username = job_state['username']
mod_id = job_state['mod_id']
run_name = job_state['run_name']
mod_name = job_state['mod_name']
args = (username, mod_name, mod_id, run_name)
run_dir = os.path.join(pce_root, 'users/%s/%s_%d/%s' % args)
ret_dir = os.getcwd()
os.chdir(run_dir)
_logger.debug('Calling bin/onramp_postprocess.py')
try:
result = check_output([os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_postprocess.py'], stderr=STDOUT)
except CalledProcessError as e:
code = e.returncode
if code > 127:
code -= 256
result = e.output
msg = ('Postprocess exited with return status %d and output: %s'
% (code, result))
with JobState(job_id) as job_state:
job_state['state'] = 'Postprocess failed'
job_state['error'] = msg
_logger.error(msg)
os.chdir(ret_dir)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (-1, msg)
finally:
module_log(run_dir, 'postprocess', result)
# Grab job output.
with open('output.txt', 'r') as f:
output = f.read()
os.chdir(ret_dir)
# Update state.
with JobState(job_id) as job_state:
job_state['state'] = 'Done'
job_state['error'] = None
job_state['output'] = output
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
def _get_module_status_output(job_id, job_state_file=None):
"""Run bin/onramp_status.py for job and return any output.
Args:
job_id (int): Id of the job to launch bin/onramp_status.py for.
Returns:
String containint output to stdout and stderr frob job's
bin/onramp_status.py script.
"""
# Get attrs needed.
with JobState(job_id, job_state_file) as job_state:
username = job_state['username']
mod_id = job_state['mod_id']
run_name = job_state['run_name']
mod_name = job_state['mod_name']
run_dir = job_state['run_dir']
#args = (username, mod_name, mod_id, run_name)
#run_dir = os.path.join(pce_root, 'users/%s/%s_%d/%s' % args)
ret_dir = os.getcwd()
# Run bin/onramp_status.py and grab output.
os.chdir(run_dir)
_logger.debug('Calling bin/onramp_status.py')
try:
output = check_output([
os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_status.py'
],
stderr=STDOUT)
except CalledProcessError as e:
code = e.returncode
if code > 127:
code -= 256
output = ('Status exited with return status %d and output: %s' %
(code, e.output))
module_log(run_dir, 'status', output)
os.chdir(ret_dir)
return output
def job_preprocess(job_id, job_state_file=None):
ret_dir = os.getcwd()
_logger.info('Calling bin/onramp_preprocess.py')
_logger.debug('Want JobState (preprocess) at: %s' % time.time())
with JobState(job_id, job_state_file) as job_state:
_logger.debug('In JobState (preprocess) at: %s' % time.time())
_logger.debug('preprocess PID: %d' % os.getpid())
job_state['state'] = 'Preprocessing'
job_state['error'] = None
run_dir = job_state['run_dir']
os.chdir(run_dir)
_logger.debug('Done with JobState (preprocess) at: %s' % time.time())
try:
result = check_output([os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_preprocess.py'], stderr=STDOUT)
except CalledProcessError as e:
code = e.returncode
if code > 127:
code -= 256
result = e.output
msg = ('Preprocess exited with return status %d and output: %s'
% (code, result))
with JobState(job_id, job_state_file) as job_state:
job_state['state'] = 'Preprocess failed'
job_state['error'] = msg
_logger.error(msg)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (-1, msg)
finally:
module_log(run_dir, 'preprocess', result)
os.chdir(ret_dir)
return (0, 'Job preprocess complete')
def _get_module_status_output(job_id):
"""Run bin/onramp_status.py for job and return any output.
Args:
job_id (int): Id of the job to launch bin/onramp_status.py for.
Returns:
String containint output to stdout and stderr frob job's
bin/onramp_status.py script.
"""
# Get attrs needed.
with JobState(job_id) as job_state:
username = job_state['username']
mod_id = job_state['mod_id']
run_name = job_state['run_name']
mod_name = job_state['mod_name']
args = (username, mod_name, mod_id, run_name)
run_dir = os.path.join(pce_root, 'users/%s/%s_%d/%s' % args)
ret_dir = os.getcwd()
# Run bin/onramp_status.py and grab output.
os.chdir(run_dir)
_logger.debug('Calling bin/onramp_status.py')
try:
output = check_output([os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_status.py'], stderr=STDOUT)
except CalledProcessError as e:
code = e.returncode
if code > 127:
code -= 256
output = ('Status exited with return status %d and output: %s'
% (code, e.output))
module_log(run_dir, 'status', output)
os.chdir(ret_dir)
return output
def deploy_module(mod_id, verbose=False):
"""Deploy an installed OnRamp educational module.
Args:
mod_id (int): Id of the module to be deployed.
Kwargs:
verbose (bool): Increases status output if True.
Returns:
Tuple with 0th position being error code and 1st position being string
indication of status.
"""
mod_dir = None
not_installed_states = ['Available', 'Checkout in Progress',
'Checkout failed']
with ModState(mod_id) as mod_state:
if ('state' not in mod_state.keys()
or mod_state['state'] in not_installed_states):
return (-1, 'Module %d not installed' % mod_id)
if mod_state['state'] == 'Deploy in progress':
return (-1, 'Deployment already underway for module %d' % mod_id)
mod_state['state'] = 'Deploy in progress'
mod_state['error'] = None
mod_dir = mod_state['installed_path']
ret_dir = os.getcwd()
_logger.debug('ret_dir: %s' % ret_dir)
os.chdir(mod_dir)
try:
_logger.debug('Calling bin/onramp_deploy.py')
_logger.debug('CWD: %s' % os.getcwd())
output = check_output([os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_deploy.py'], stderr=STDOUT)
_logger.debug('Back from bin/onramp_deploy.py')
except CalledProcessError as e:
_logger.debug('CalledProcessError from bin/onramp_deploy.py')
code = e.returncode
if code > 127:
code -= 256
output = e.output
if code != 1:
with ModState(mod_id) as mod_state:
msg = ('Deploy exited with return status %d and output: %s'
% (code, output))
_logger.debug(msg)
mod_state['state'] = 'Deploy failed'
mod_state['error'] = msg
if mod_state['_marked_for_del']:
_delete_module(mod_state)
return (-3, 'Module %d deleted' % mod_id)
return (-1, msg)
with ModState(mod_id) as mod_state:
msg = 'Admin required'
_logger.debug(msg)
mod_state['state'] = msg
mod_state['error'] = output
if mod_state['_marked_for_del']:
_delete_module(mod_state)
return (-3, 'Module %d deleted' % mod_id)
return (1, msg)
except OSError as e1:
output = str(e1)
_logger.debug('OSError from bin/onramp_deploy.py')
_logger.debug(e1)
with ModState(mod_id) as mod_state:
mod_state['state'] = 'Deploy failed'
mod_state['error'] = str(e1)
return (-1, str(e1))
finally:
os.chdir(ret_dir)
module_log(mod_dir, 'deploy', output)
_logger.debug("Updating state to 'Module ready'")
with ModState(mod_id) as mod_state:
mod_state['state'] = 'Module ready'
mod_state['error'] = None
if mod_state['_marked_for_del']:
_delete_module(mod_state)
return (-3, 'Module %d deleted' % mod_id)
return (0, 'Module %d ready' % mod_id)
def deploy_module(mod_id, verbose=False, mod_state_file=None):
"""Deploy an installed OnRamp educational module.
Args:
mod_id (int): Id of the module to be deployed.
Kwargs:
verbose (bool): Increases status output if True.
Returns:
Tuple with 0th position being error code and 1st position being string
indication of status.
"""
mod_dir = None
not_installed_states = ["Available", "Checkout in Progress", "Checkout failed"]
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
if "state" not in mod_state.keys() or mod_state["state"] in not_installed_states:
return (-1, "Module %d not installed" % mod_id)
if mod_state["state"] == "Deploy in progress":
return (-1, "Deployment already underway for module %d" % mod_id)
mod_state["state"] = "Deploy in progress"
mod_state["error"] = None
mod_dir = mod_state["installed_path"]
ret_dir = os.getcwd()
_logger.debug("ret_dir: %s" % ret_dir)
os.chdir(mod_dir)
try:
_logger.debug("Calling bin/onramp_deploy.py")
_logger.debug("CWD: %s" % os.getcwd())
output = check_output([os.path.join(pce_root, "src/env/bin/python"), "bin/onramp_deploy.py"], stderr=STDOUT)
_logger.debug("Back from bin/onramp_deploy.py")
except CalledProcessError as e:
_logger.debug("CalledProcessError from bin/onramp_deploy.py")
code = e.returncode
if code > 127:
code -= 256
output = e.output
if code != 1:
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
msg = "Deploy exited with return status %d and output: %s" % (code, output)
_logger.debug(msg)
mod_state["state"] = "Deploy failed"
mod_state["error"] = msg
if mod_state["_marked_for_del"]:
_delete_module(mod_state)
return (-3, "Module %d deleted" % mod_id)
return (-1, msg)
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
msg = "Admin required"
_logger.debug(msg)
mod_state["state"] = msg
mod_state["error"] = output
if mod_state["_marked_for_del"]:
_delete_module(mod_state)
return (-3, "Module %d deleted" % mod_id)
return (1, msg)
except OSError as e1:
output = str(e1)
_logger.debug("OSError from bin/onramp_deploy.py")
_logger.debug(e1)
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
mod_state["state"] = "Deploy failed"
mod_state["error"] = str(e1)
return (-1, str(e1))
finally:
os.chdir(ret_dir)
module_log(mod_dir, "deploy", output)
_logger.debug("Updating state to 'Module ready'")
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
mod_state["state"] = "Module ready"
mod_state["error"] = None
if mod_state["_marked_for_del"]:
_delete_module(mod_state)
return (-3, "Module %d deleted" % mod_id)
return (0, "Module %d ready" % mod_id)
def launch_job(job_id, mod_id, username, run_name, run_params):
"""Schedule job launch using system batch scheduler as configured in
onramp_pce_config.cfg.
Args:
job_id (int): Unique identifier for job.
mod_id (int): Id for OnRamp educational module to run in this job.
username (str): Username of user running the job.
run_name (str): Human-readable label for this job run.
Returns:
Tuple with 0th position being error code and 1st position being string
indication of status.
"""
accepted_states = ['Schedule failed', 'Launch failed', 'Preprocess failed']
_logger.debug('PCE.tools.launch_job() called')
# Initialize job state.
with JobState(job_id) as job_state:
if ('state' in job_state.keys()
and job_state['state'] not in accepted_states):
msg = 'Job launch already initiated'
_logger.warn(msg)
return (-1, msg)
job_state['job_id'] = job_id
job_state['mod_id'] = mod_id
job_state['username'] = username
job_state['run_name'] = run_name
job_state['scheduler_job_num'] = None
job_state['state'] = 'Setting up launch'
job_state['error'] = None
job_state['mod_status_output'] = None
job_state['output'] = None
job_state['visible_files'] = None
job_state['mod_name'] = None
job_state['_marked_for_del'] = False
_logger.debug('Waiting on ModState at: %s' % time.time())
with ModState(mod_id) as mod_state:
_logger.debug('Done waiting on ModState at: %s' % time.time())
if ('state' not in mod_state.keys()
or mod_state['state'] != 'Module ready'):
msg = 'Module not ready'
job_state['state'] = 'Launch failed'
job_state['error'] = msg
_logger.warn(msg)
_logger.warn('mod_state: %s' % str(mod_state))
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (-1, 'Module not ready')
job_state['mod_name'] = mod_state['mod_name']
proj_loc = mod_state['installed_path']
mod_name = mod_state['mod_name']
_logger.debug('Testing project location')
if not os.path.isdir(proj_loc):
msg = 'Project location does not exist'
_logger.error(msg)
return (-1, msg)
_logger.debug('Project location good')
# Initialize dir structure.
user_dir = os.path.join(os.path.join(pce_root, 'users'), username)
user_mod_dir = os.path.join(user_dir, '%s_%d' % (mod_name, mod_id))
run_dir = os.path.join(user_mod_dir, run_name)
try:
os.mkdir(user_dir)
except OSError:
# Thrown if dir already exists.
pass
try:
os.mkdir(user_mod_dir)
except OSError:
# Thrown if dir already exists.
pass
# The way the following is setup, if a run_dir has already been setup with
# this run_name, it will be used (that is, not overwritten) for this launch.
try:
shutil.copytree(proj_loc, run_dir)
except shutil.Error as e:
pass
if run_params:
_logger.debug('Handling run_params')
spec = os.path.join(run_dir, 'config/onramp_uioptions.cfgspec')
params = ConfigObj(run_params, configspec=spec)
result = params.validate(Validator())
if result:
with open(os.path.join(run_dir, 'onramp_runparams.cfg'), 'w') as f:
params.write(f)
else:
msg = 'Runparams failed validation'
_logger.warn(msg)
return (-1, msg)
ret_dir = os.getcwd()
os.chdir(run_dir)
# Preprocess.
_logger.info('Calling bin/onramp_preprocess.py')
with JobState(job_id) as job_state:
job_state['state'] = 'Preprocessing'
job_state['error'] = None
try:
result = check_output([os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_preprocess.py'], stderr=STDOUT)
except CalledProcessError as e:
code = e.returncode
if code > 127:
code -= 256
result = e.output
msg = ('Preprocess exited with return status %d and output: %s'
% (code, result))
with JobState(job_id) as job_state:
job_state['state'] = 'Preprocess failed'
job_state['error'] = msg
_logger.error(msg)
os.chdir(ret_dir)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (-1, msg)
finally:
module_log(run_dir, 'preprocess', result)
# Determine batch scheduler to user from config.
cfg = ConfigObj(os.path.join(pce_root, 'bin', 'onramp_pce_config.cfg'),
configspec=os.path.join(pce_root, 'src', 'configspecs',
'onramp_pce_config.cfgspec'))
cfg.validate(Validator())
scheduler = Scheduler(cfg['cluster']['batch_scheduler'])
# Write batch script.
with open('script.sh', 'w') as f:
f.write(scheduler.get_batch_script(run_name))
# Schedule job.
result = scheduler.schedule(run_dir)
if result['status_code'] != 0:
_logger.error(result['msg'])
with JobState(job_id) as job_state:
job_state['state'] = 'Schedule failed'
job_state['error'] = result['msg']
os.chdir(ret_dir)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (result['returncode'], result['msg'])
with JobState(job_id) as job_state:
job_state['state'] = 'Scheduled'
job_state['error'] = None
job_state['scheduler_job_num'] = result['job_num']
os.chdir(ret_dir)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (0, 'Job scheduled')
def deploy_module(mod_id, verbose=False, mod_state_file=None):
"""Deploy an installed OnRamp educational module.
Args:
mod_id (int): Id of the module to be deployed.
Kwargs:
verbose (bool): Increases status output if True.
Returns:
Tuple with 0th position being error code and 1st position being string
indication of status.
"""
mod_dir = None
not_installed_states = [
'Available', 'Checkout in Progress', 'Checkout failed'
]
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
if ('state' not in mod_state.keys()
or mod_state['state'] in not_installed_states):
return (-1, 'Module %d not installed' % mod_id)
if mod_state['state'] == 'Deploy in progress':
return (-1, 'Deployment already underway for module %d' % mod_id)
mod_state['state'] = 'Deploy in progress'
mod_state['error'] = None
mod_dir = mod_state['installed_path']
ret_dir = os.getcwd()
_logger.debug('ret_dir: %s' % ret_dir)
os.chdir(mod_dir)
try:
_logger.debug('Calling bin/onramp_deploy.py')
_logger.debug('CWD: %s' % os.getcwd())
output = check_output([
os.path.join(pce_root, 'src/env/bin/python'),
'bin/onramp_deploy.py'
],
stderr=STDOUT)
_logger.debug('Back from bin/onramp_deploy.py')
except CalledProcessError as e:
_logger.debug('CalledProcessError from bin/onramp_deploy.py')
code = e.returncode
if code > 127:
code -= 256
output = e.output
if code != 1:
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
msg = ('Deploy exited with return status %d and output: %s' %
(code, output))
_logger.debug(msg)
mod_state['state'] = 'Deploy failed'
mod_state['error'] = msg
if mod_state['_marked_for_del']:
_delete_module(mod_state)
return (-3, 'Module %d deleted' % mod_id)
return (-1, msg)
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
msg = 'Admin required'
_logger.debug(msg)
mod_state['state'] = msg
mod_state['error'] = output
if mod_state['_marked_for_del']:
_delete_module(mod_state)
return (-3, 'Module %d deleted' % mod_id)
return (1, msg)
except OSError as e1:
output = str(e1)
_logger.debug('OSError from bin/onramp_deploy.py')
_logger.debug(e1)
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
mod_state['state'] = 'Deploy failed'
mod_state['error'] = str(e1)
return (-1, str(e1))
finally:
os.chdir(ret_dir)
module_log(mod_dir, 'deploy', output)
_logger.debug("Updating state to 'Module ready'")
with ModState(mod_id, mod_state_file=mod_state_file) as mod_state:
mod_state['state'] = 'Module ready'
mod_state['error'] = None
if mod_state['_marked_for_del']:
_delete_module(mod_state)
return (-3, 'Module %d deleted' % mod_id)
return (0, 'Module %d ready' % mod_id)
