def _delete_job(job_state):
"""Delete given job.
Both state for and contents of job will be removed.
Args:
job_state (JobState): State object for the job to remove.
"""
job_cancel_states = ['Scheduled', 'Queued', 'Running']
if job_state['state'] in job_cancel_states:
cfgfile = os.path.join(pce_root, 'bin', 'onramp_pce_config.cfg')
specfile = os.path.join(pce_root, 'src', 'configspecs',
'onramp_pce_config.cfgspec')
cfg = ConfigObj(cfgfile, configspec=specfile)
cfg.validate(Validator())
scheduler = Scheduler(cfg['cluster']['batch_scheduler'])
result = scheduler.cancel_job(job_state['scheduler_job_num'])
_logger.debug('Cancel job output: %s' % result[1])
job_state_file = os.path.join(_job_state_dir, str(job_state['job_id']))
os.remove(job_state_file)
args = (job_state['username'], job_state['mod_name'], job_state['mod_id'],
job_state['run_name'])
run_dir = os.path.join(pce_root, 'users/%s/%s_%d/%s' % args)
shutil.rmtree(run_dir, ignore_errors=True)
job_state.clear()
def job_run(job_id, job_state_file=None):
# Determine batch scheduler to user from config.
cfg = ConfigObj(os.path.join(pce_root, 'bin', 'onramp_pce_config.cfg'),
configspec=os.path.join(pce_root, 'src', 'configspecs',
'onramp_pce_config.cfgspec'))
cfg.validate(Validator())
scheduler = Scheduler(cfg['cluster']['batch_scheduler'])
ret_dir = os.getcwd()
with JobState(job_id, job_state_file) as job_state:
run_dir = job_state['run_dir']
run_name = job_state['run_name']
os.chdir(run_dir)
# Write batch script.
with open('script.sh', 'w') as f:
f.write(scheduler.get_batch_script(run_name))
# Schedule job.
result = scheduler.schedule(run_dir)
if result['status_code'] != 0:
_logger.error(result['msg'])
with JobState(job_id, job_state_file) as job_state:
job_state['state'] = 'Schedule failed'
job_state['error'] = result['msg']
os.chdir(ret_dir)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (result['returncode'], result['msg'])
with JobState(job_id, job_state_file) as job_state:
job_state['state'] = 'Scheduled'
job_state['error'] = None
job_state['scheduler_job_num'] = result['job_num']
os.chdir(ret_dir)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (0, 'Job scheduled')
def _build_job(job_id, job_state_file=None):
"""Launch actions required to maintain job state and/or currate job results
and return the state.
When current job state (as a function of both PCE state tracking and
scheduler output) warrants, initiate job postprocessing and/or status
checking prior to building and returning state.
Args:
job_id (int): Id of the job to get state for.
Returns:
OnRamp formatted dictionary containing job attrs.
"""
status_check_states = ['Scheduled', 'Queued', 'Running']
with JobState(job_id, job_state_file) as job_state:
_logger.debug('Building at %s' % time.time())
if 'state' not in job_state.keys():
_logger.debug('No state at %s' % time.time())
_logger.debug('job_state keys: %s' % job_state.keys())
return {}
if job_state['state'] in status_check_states:
specfile = os.path.join(pce_root, 'src', 'configspecs',
'onramp_pce_config.cfgspec')
cfg = ConfigObj(os.path.join(pce_root, 'bin',
'onramp_pce_config.cfg'),
configspec=specfile)
cfg.validate(Validator())
scheduler = Scheduler(cfg['cluster']['batch_scheduler'])
sched_job_num = job_state['scheduler_job_num']
job_status = scheduler.check_status(sched_job_num)
# Bad.
if job_status[0] != 0:
_logger.debug('Bad job status: %s' % job_status[1])
job_state['state'] = 'Run failed'
job_state['error'] = job_status[1]
if job_status[0] != -2:
job_state['state'] = job_status[1]
if job_state['_marked_for_del']:
_delete_job(job_state)
# FIXME: This might cause trouble. About to return {}.
return copy.deepcopy(job_state)
return copy.deepcopy(job_state)
# Good.
if job_status[1] in ['Done', 'No info']:
job_state['state'] = 'Postprocessing'
if job_state['_marked_for_del']:
_delete_job(job_state)
# FIXME: This might cause trouble. About to return {}.
return copy.deepcopy(job_state)
job_state['error'] = None
job_state['mod_status_output'] = None
p = Process(target=job_postprocess,
args=(job_id, job_state_file))
p.start()
elif job_status[1] == 'Running':
job_state['state'] = 'Running'
job_state['error'] = None
if job_state['_marked_for_del']:
_delete_job(job_state)
# FIXME: This might cause trouble. About to return {}.
return copy.deepcopy(job_state)
run_dir = job_state['run_dir']
mod_status_output = _get_module_status_output(run_dir)
job_state['mod_status_output'] = mod_status_output
elif job_status[1] == 'Queued':
job_state['state'] = 'Queued'
job_state['error'] = None
if job_state['_marked_for_del']:
_delete_job(job_state)
# FIXME: This might cause trouble. About to return {}.
return copy.deepcopy(job_state)
job = copy.deepcopy(job_state)
if job['state'] in ['Launch failed', 'Setting up launch']:
return job
# Build visible files.
_logger.debug('job state: %s' % str(job))
dir_args = (job['username'], job['mod_name'], job['mod_id'],
job['run_name'])
run_dir = os.path.join(pce_root, 'users/%s/%s_%d/%s' % dir_args)
cfg_file = os.path.join(run_dir, 'config/onramp_metadata.cfg')
try:
conf = ConfigObj(cfg_file, file_error=True)
except (IOError, SyntaxError):
# Badly formed or non-existant config/onramp_metadata.cfg.
_logger.debug('Bad metadata')
_logger.debug(cfg_file)
return job
if 'onramp' in conf.keys() and 'visible' in conf['onramp'].keys():
globs = conf['onramp']['visible']
if isinstance(globs, basestring):
# Globs is only a single string. Convert to list.
globs = [globs]
else:
globs = []
ret_dir = os.getcwd()
os.chdir(run_dir)
filenames = [name for name in chain.from_iterable(map(glob.glob, globs))]
prefix = os.path.join(pce_root, 'users') + '/'
url_prefix = run_dir.split(prefix)[1]
job['visible_files'] = [{
'name':
filename,
'size':
os.path.getsize(os.path.join(run_dir, filename)),
'url':
os.path.join('files', os.path.join(url_prefix, filename))
} for filename in filenames]
os.chdir(ret_dir)
return job
def job_run(job_id, job_state_file=None):
# Determine batch scheduler to user from config.
cfg = ConfigObj(os.path.join(pce_root, 'bin', 'onramp_pce_config.cfg'),
configspec=os.path.join(pce_root, 'src', 'configspecs',
'onramp_pce_config.cfgspec'))
cfg.validate(Validator())
scheduler = Scheduler(cfg['cluster']['batch_scheduler'])
_logger.debug("in job_run: trying to launch using scheduler %s", cfg['cluster']['batch_scheduler'])
#ret_dir = os.getcwd()
with JobState(job_id, job_state_file) as job_state:
run_dir = job_state['run_dir']
run_name = job_state['run_name']
os.chdir(run_dir)
#_logger.debug("in job_run: attempting to be in %s, really in %s", run_dir, os.get_cwd())
# Load run params:
run_np = None
run_nodes = None
run_cfg = ConfigObj('onramp_runparams.cfg')
if 'onramp' in run_cfg.keys():
if 'np' in run_cfg['onramp']:
run_np = run_cfg['onramp']['np']
if 'nodes' in run_cfg['onramp']:
run_nodes = run_cfg['onramp']['nodes']
_logger.debug("in job_run: loaded params np: %d and nodes: %d", run_np, run_nodes)
# Write batch script.
with open('script.sh', 'w') as f:
if run_np and run_nodes:
f.write(scheduler.get_batch_script(run_name, numtasks=run_np,
num_nodes=run_nodes))
elif run_np:
f.write(scheduler.get_batch_script(run_name, numtasks=run_np))
elif run_nodes:
f.write(scheduler.get_batch_script(run_name, num_nodes=run_nodes))
else:
f.write(scheduler.get_batch_script(run_name))
# Schedule job.
result = scheduler.schedule(run_dir)
if result['status_code'] != 0:
_logger.error(result['msg'])
with JobState(job_id, job_state_file) as job_state:
job_state['state'] = 'Schedule failed'
job_state['error'] = result['msg']
os.chdir(ret_dir)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (result['returncode'], result['msg'])
with JobState(job_id, job_state_file) as job_state:
job_state['state'] = 'Scheduled'
job_state['error'] = None
job_state['scheduler_job_num'] = result['job_num']
os.chdir(ret_dir)
if job_state['_marked_for_del']:
_delete_job(job_state)
return (-2, 'Job %d deleted' % job_id)
return (0, 'Job scheduled')
