def update_paths(spec):
"""
This functiojn will update all the paths in spec that need the current working directory.
"""
for key in spec:
value = spec[key]
# if path needs current working directory then
if type(value) == type(""):
if value.find(client_utils.CWD_TAG) != -1:
if 'cwd' in spec:
_cwd = spec['cwd']
else:
_cwd = client_utils.getcwd()
_path = spec[key].replace(client_utils.CWD_TAG,_cwd)
# validate the path
if not os.path.isdir(os.path.dirname(_path)):
client_utils.logger.error("directory %s does not exist" % _path)
sys.exit(1)
spec[key] = _path
def update_paths(spec):
"""
This functiojn will update all the paths in spec that need the current working directory.
"""
for key in spec:
value = spec[key]
# if path needs current working directory then
if type(value) == type(""):
if value.find(client_utils.CWD_TAG) != -1:
if 'cwd' in spec:
_cwd = spec['cwd']
else:
_cwd = client_utils.getcwd()
_path = spec[key].replace(client_utils.CWD_TAG, _cwd)
# validate the path
if not os.path.isdir(os.path.dirname(_path)):
client_utils.logger.error("directory %s does not exist" %
_path)
sys.exit(1)
spec[key] = _path
def main():
"""
qsub main function.
"""
# setup logging for client. The clients should call this before doing anything else.
client_utils.setup_logging(logging.INFO)
spec = {} # map of destination option strings and parsed values
opts = {} # old map
opt2spec = {}
def_spec = {}
# list of callback with its arguments
callbacks = [
# <cb function> <cb args (tuple) >
( cb_debug , () ),
( cb_interactive , () ),
( cb_env , (opts,) ),
( cb_nodes , (False,) ), # return string
( cb_gtzero , (False,) ), # return string
( cb_time , (False, False, False) ), # no delta time, minutes, return string
( cb_umask , () ),
( cb_path , (opts, True) ), # use CWD
( cb_dep , () ),
( cb_attrs , () ),
( cb_mode , () ),
( cb_user_list , (opts,) ),
( cb_geometry , (opts,) )]
# Get the version information
opt_def = __doc__.replace('__revision__', __revision__)
opt_def = opt_def.replace('__version__', __version__)
user = client_utils.getuid()
def_spec['tag'] = 'job'
def_spec['user'] = user
def_spec['outputdir'] = client_utils.CWD_TAG
def_spec['jobid'] = '*'
def_spec['path'] = client_utils.getpath()
def_spec['mode'] = False
def_spec['cwd'] = client_utils.getcwd()
def_spec['kernel'] = CN_DEFAULT_KERNEL
def_spec['ion_kernel'] = ION_DEFAULT_KERNEL
def_spec['queue'] = 'default'
def_spec['umask'] = 022
def_spec['run_project'] = False
def_spec['user_list'] = [user]
def_spec['procs'] = False
def_spec['script_preboot'] = True
parser = ArgParse(opt_def, callbacks)
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
reparse = validate_args(parser, spec)
if reparse:
# re-parse with new sys.argv
# note: the first parse is necessary to make sure that
# the env syntax is correct for every --env option provided
# If not parsed prior to the union then the union could result
# in a valid syntax, but it would not be what the user would want.
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
client_utils.setumask(spec['umask'])
validate_options(parser, opt_count)
update_outputprefix(parser, spec)
update_paths(spec)
check_inputfile(parser, spec)
not_exit_on_interrupt()
opts['qsub_host'] = socket.gethostname()
opts = client_utils.component_call(SYSMGR, False, 'validate_job',(opts,))
impl = client_utils.component_call(SYSMGR, False, 'get_implementation',())
exit_on_interrupt()
if impl in ['alps_system']:
# If we're interactive, remote and go.
if opts['mode'] == 'interactive':
if opts.get('ssh_host', None) is not None:
if opts['qsub_host'] != opts['ssh_host']:
#remote execute qsub on the ssh_host
client_utils.logger.info('Connecting to %s for interactive qsub...', opts['ssh_host'])
sys.exit(qsub_remote_host(opts['ssh_host'])) # return status from qsub-ssh
filters = client_utils.get_filters()
client_utils.process_filters(filters, spec)
# Attrs needs special handling filter won't set otherwise
if spec.get('attrs', None) is not None:
opts['attrs'].update(ast.literal_eval(str(spec['attrs'])))
update_spec(parser, opts, spec, opt2spec)
run_job(parser, user, spec, opts)
def main():
"""
qsub main function.
"""
# setup logging for client. The clients should call this before doing anything else.
client_utils.setup_logging(logging.INFO)
spec = {} # map of destination option strings and parsed values
opts = {} # old map
opt2spec = {}
def_spec = {}
# list of callback with its arguments
callbacks = [
# <cb function> <cb args (tuple) >
(cb_debug, ()),
(cb_interactive, ()),
(cb_env, (opts, )),
(cb_nodes, (False, )), # return string
(cb_gtzero, (False, )), # return string
(cb_time, (False, False,
False)), # no delta time, minutes, return string
(cb_umask, ()),
(cb_path, (opts, True)), # use CWD
(cb_dep, ()),
(cb_attrs, ()),
(cb_mode, ()),
(cb_user_list, (opts, )),
(cb_geometry, (opts, ))
]
# Get the version information
opt_def = __doc__.replace('__revision__', __revision__)
opt_def = opt_def.replace('__version__', __version__)
user = client_utils.getuid()
def_spec['tag'] = 'job'
def_spec['user'] = user
def_spec['outputdir'] = client_utils.CWD_TAG
def_spec['jobid'] = '*'
def_spec['path'] = client_utils.getpath()
def_spec['mode'] = False
def_spec['cwd'] = client_utils.getcwd()
def_spec['kernel'] = CN_DEFAULT_KERNEL
def_spec['ion_kernel'] = ION_DEFAULT_KERNEL
def_spec['queue'] = 'default'
def_spec['umask'] = 022
def_spec['run_project'] = False
def_spec['user_list'] = [user]
def_spec['procs'] = False
def_spec['script_preboot'] = True
parser = ArgParse(opt_def, callbacks)
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
reparse = validate_args(parser, spec)
if reparse:
# re-parse with new sys.argv
# note: the first parse is necessary to make sure that
# the env syntax is correct for every --env option provided
# If not parsed prior to the union then the union could result
# in a valid syntax, but it would not be what the user would want.
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
client_utils.setumask(spec['umask'])
validate_options(parser, opt_count)
update_outputprefix(parser, spec)
update_paths(spec)
check_inputfile(parser, spec)
not_exit_on_interrupt()
opts = client_utils.component_call(SYSMGR, False, 'validate_job', (opts, ))
exit_on_interrupt()
filters = client_utils.get_filters()
client_utils.process_filters(filters, spec)
update_spec(parser, opts, spec, opt2spec)
run_job(parser, user, spec, opts)
def main():
"""
qsub main function.
"""
# setup logging for client. The clients should call this before doing anything else.
client_utils.setup_logging(logging.INFO)
spec = {} # map of destination option strings and parsed values
opts = {} # old map
opt2spec = {}
def_spec = {}
# list of callback with its arguments
callbacks = [
# <cb function> <cb args (tuple) >
( cb_debug , () ),
( cb_interactive , () ),
( cb_env , (opts,) ),
( cb_nodes , (False,) ), # return string
( cb_gtzero , (False,) ), # return string
( cb_time , (False, False, False) ), # no delta time, minutes, return string
( cb_umask , () ),
( cb_path , (opts, True) ), # use CWD
( cb_dep , () ),
( cb_attrs , () ),
( cb_mode , () ),
( cb_user_list , (opts,) ),
( cb_geometry , (opts,) )]
# Get the version information
opt_def = __doc__.replace('__revision__', __revision__)
opt_def = opt_def.replace('__version__', __version__)
user = client_utils.getuid()
def_spec['tag'] = 'job'
def_spec['user'] = user
def_spec['outputdir'] = client_utils.CWD_TAG
def_spec['jobid'] = '*'
def_spec['path'] = client_utils.getpath()
def_spec['mode'] = False
def_spec['cwd'] = client_utils.getcwd()
def_spec['kernel'] = CN_DEFAULT_KERNEL
def_spec['ion_kernel'] = ION_DEFAULT_KERNEL
def_spec['queue'] = 'default'
def_spec['umask'] = 022
def_spec['run_project'] = False
def_spec['user_list'] = [user]
def_spec['procs'] = False
def_spec['script_preboot'] = True
parser = ArgParse(opt_def, callbacks)
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
reparse = validate_args(parser, spec)
if reparse:
# re-parse with new sys.argv
# note: the first parse is necessary to make sure that
# the env syntax is correct for every --env option provided
# If not parsed prior to the union then the union could result
# in a valid syntax, but it would not be what the user would want.
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
client_utils.setumask(spec['umask'])
validate_options(parser, opt_count)
update_outputprefix(parser, spec)
update_paths(spec)
check_inputfile(parser, spec)
not_exit_on_interrupt()
opts = client_utils.component_call(SYSMGR, False, 'validate_job',(opts,))
exit_on_interrupt()
filters = client_utils.get_filters()
client_utils.process_filters(filters, spec)
update_spec(parser, opts, spec, opt2spec)
run_job(parser, user, spec, opts)
def main():
"""
qsub main function.
"""
# setup logging for client. The clients should call this before doing anything else.
client_utils.setup_logging(logging.INFO)
spec = {} # map of destination option strings and parsed values
opts = {} # old map
opt2spec = {}
def_spec = {}
# list of callback with its arguments
callbacks = [
# <cb function> <cb args (tuple) >
( cb_debug , () ),
( cb_env , (opts,) ),
( cb_nodes , (False,) ), # return string
( cb_gtzero , (False,) ), # return string
( cb_time , (False, False, False) ), # no delta time, minutes, return string
( cb_umask , () ),
( cb_path , (opts, True) ), # use CWD
( cb_dep , () ),
( cb_attrs , () ),
( cb_mode , () ),
( cb_user_list , (opts,) ),
( cb_geometry , (opts,) )]
# Get the version information
opt_def = __doc__.replace('__revision__', __revision__)
opt_def = opt_def.replace('__version__', __version__)
user = client_utils.getuid()
def_spec['tag'] = 'job'
def_spec['user'] = user
def_spec['outputdir'] = client_utils.CWD_TAG
def_spec['jobid'] = '*'
def_spec['path'] = client_utils.getpath()
def_spec['mode'] = False
def_spec['cwd'] = client_utils.getcwd()
def_spec['kernel'] = CN_DEFAULT_KERNEL
def_spec['ion_kernel'] = ION_DEFAULT_KERNEL
def_spec['queue'] = 'default'
def_spec['umask'] = 022
def_spec['run_project'] = False
def_spec['user_list'] = [user]
def_spec['procs'] = False
def_spec['script_preboot'] = True
parser = ArgParse(opt_def, callbacks)
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
reparse = validate_args(parser, spec)
if reparse:
# re-parse with new sys.argv
# note: the first parse is necessary to make sure that
# the env syntax is correct for every --env option provided
# If not parsed prior to the union then the union could result
# in a valid syntax, but it would not be what the user would want.
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
client_utils.setumask(spec['umask'])
validate_options(parser, opt_count)
update_outputprefix(parser, spec)
update_paths(spec)
check_inputfile(parser, spec)
opts = client_utils.component_call(SYSMGR, False, 'validate_job',(opts,))
filters = client_utils.get_filters()
client_utils.process_filters(filters, spec)
update_spec(opts, spec, opt2spec)
get_interactive_command(parser, spec, opts, opt2spec, def_spec)
jobs = client_utils.component_call(QUEMGR, False, 'add_jobs',([spec],))
def on_interrupt(sig, func=None):
'''Handler to cleanup the queued 'dummy' job if the user interrupts
qsub -I forcibly
'''
try:
spec = [{'tag':'job', 'jobid':jobs[0]['jobid'], 'user':user}]
except NameError:
sys.exit(1)
except:
raise
client_utils.logger.info("Deleting job %d", (jobs[0]['jobid']))
del_jobs = client_utils.component_call(QUEMGR, False, 'del_jobs', (spec, False, user))
client_utils.logger.info("%s", del_jobs)
sys.exit(1)
#reset sigint and sigterm interrupt handlers to deal with interactive failures
signal.signal(signal.SIGINT, on_interrupt)
signal.signal(signal.SIGTERM, on_interrupt)
if parser.options.envs:
client_utils.logger.debug("Environment Vars: %s", parser.options.envs)
logjob(jobs[0], spec)
# If this is an interactive job, wait for it to start, then ssh in
if parser.options.interactive:
headnode = ""
query = [{'tag':'job', 'jobid':jobs[0]['jobid'], 'location':'*', 'state':"*"}]
while True:
response = client_utils.component_call(QUEMGR, False, 'get_jobs', (query, ))
state = response[0]['state']
location = response[0]['location']
if state == 'running' and location:
headnode = location[0]
client_utils.logger.info("Opening ssh session to %s", headnode)
break
elif state != 'running' and state != 'queued' and state != 'starting':
client_utils.logger.error("ERROR: Something went wrong with job submission, did not expect job to reach %s state.",
response[0]['state'])
break
else:
#using Cobalt Utils sleep here
sleep(2)
if not headnode:
client_utils.logger.error("Unable to determine head node for job %d", (jobs[0]['jobid']))
else:
try:
os.putenv("COBALT_NODEFILE", "/var/tmp/cobalt.%s" % (response[0]['jobid']))
os.putenv("COBALT_JOBID", "%s" % (response[0]['jobid']))
os.system("/usr/bin/ssh -o \"SendEnv COBALT_NODEFILE COBALT_JOBID\" %s" % (headnode))
except:
client_utils.logger.error('Exception occurred during execution ssh session. Job Terminated.')
spec = [{'tag':'job', 'jobid':jobs[0]['jobid'], 'user':user}]
jobs = client_utils.component_call(QUEMGR, False, 'del_jobs', (spec, False, user))
def main():
"""
qsub main function.
"""
# setup logging for client. The clients should call this before doing anything else.
client_utils.setup_logging(logging.INFO)
spec = {} # map of destination option strings and parsed values
opts = {} # old map
opt2spec = {}
def_spec = {}
# list of callback with its arguments
callbacks = [
# <cb function> <cb args (tuple) >
(cb_debug, ()),
(cb_env, (opts, )),
(cb_nodes, (False, )), # return string
(cb_gtzero, (False, )), # return string
(cb_time, (False, False,
False)), # no delta time, minutes, return string
(cb_umask, ()),
(cb_path, (opts, True)), # use CWD
(cb_dep, ()),
(cb_attrs, ()),
(cb_mode, ()),
(cb_user_list, (opts, )),
(cb_geometry, (opts, ))
]
# Get the version information
opt_def = __doc__.replace('__revision__', __revision__)
opt_def = opt_def.replace('__version__', __version__)
user = client_utils.getuid()
def_spec['tag'] = 'job'
def_spec['user'] = user
def_spec['outputdir'] = client_utils.CWD_TAG
def_spec['jobid'] = '*'
def_spec['path'] = client_utils.getpath()
def_spec['mode'] = False
def_spec['cwd'] = client_utils.getcwd()
def_spec['kernel'] = CN_DEFAULT_KERNEL
def_spec['ion_kernel'] = ION_DEFAULT_KERNEL
def_spec['queue'] = 'default'
def_spec['umask'] = 022
def_spec['run_project'] = False
def_spec['user_list'] = [user]
def_spec['procs'] = False
def_spec['script_preboot'] = True
parser = ArgParse(opt_def, callbacks)
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
reparse = validate_args(parser, spec)
if reparse:
# re-parse with new sys.argv
# note: the first parse is necessary to make sure that
# the env syntax is correct for every --env option provided
# If not parsed prior to the union then the union could result
# in a valid syntax, but it would not be what the user would want.
opt_count = parse_options(parser, spec, opts, opt2spec, def_spec)
client_utils.setumask(spec['umask'])
validate_options(parser, opt_count)
update_outputprefix(parser, spec)
update_paths(spec)
check_inputfile(parser, spec)
opts = client_utils.component_call(SYSMGR, False, 'validate_job', (opts, ))
filters = client_utils.get_filters()
client_utils.process_filters(filters, spec)
update_spec(opts, spec, opt2spec)
get_interactive_command(parser, spec, opts, opt2spec, def_spec)
jobs = client_utils.component_call(QUEMGR, False, 'add_jobs', ([spec], ))
def on_interrupt(sig, func=None):
'''Handler to cleanup the queued 'dummy' job if the user interrupts
qsub -I forcibly
'''
try:
spec = [{'tag': 'job', 'jobid': jobs[0]['jobid'], 'user': user}]
except NameError:
sys.exit(1)
except:
raise
client_utils.logger.info("Deleting job %d", (jobs[0]['jobid']))
del_jobs = client_utils.component_call(QUEMGR, False, 'del_jobs',
(spec, False, user))
client_utils.logger.info("%s", del_jobs)
sys.exit(1)
#reset sigint and sigterm interrupt handlers to deal with interactive failures
signal.signal(signal.SIGINT, on_interrupt)
signal.signal(signal.SIGTERM, on_interrupt)
if parser.options.envs:
client_utils.logger.debug("Environment Vars: %s", parser.options.envs)
logjob(jobs[0], spec)
# If this is an interactive job, wait for it to start, then ssh in
if parser.options.interactive:
headnode = ""
query = [{
'tag': 'job',
'jobid': jobs[0]['jobid'],
'location': '*',
'state': "*"
}]
while True:
response = client_utils.component_call(QUEMGR, False, 'get_jobs',
(query, ))
state = response[0]['state']
location = response[0]['location']
if state == 'running' and location:
headnode = location[0]
client_utils.logger.info("Opening ssh session to %s", headnode)
break
elif state != 'running' and state != 'queued' and state != 'starting':
client_utils.logger.error(
"ERROR: Something went wrong with job submission, did not expect job to reach %s state.",
response[0]['state'])
break
else:
#using Cobalt Utils sleep here
sleep(2)
if not headnode:
client_utils.logger.error(
"Unable to determine head node for job %d", (jobs[0]['jobid']))
else:
try:
os.putenv("COBALT_NODEFILE",
"/var/tmp/cobalt.%s" % (response[0]['jobid']))
os.putenv("COBALT_JOBID", "%s" % (response[0]['jobid']))
os.system(
"/usr/bin/ssh -o \"SendEnv COBALT_NODEFILE COBALT_JOBID\" %s"
% (headnode))
except:
client_utils.logger.error(
'Exception occurred during execution ssh session. Job Terminated.'
)
spec = [{'tag': 'job', 'jobid': jobs[0]['jobid'], 'user': user}]
jobs = client_utils.component_call(QUEMGR, False, 'del_jobs',
(spec, False, user))