def main_do_psuade(args=None):
"""Takes a PSUADE file and submits all the jobs contained therein as a new session. Waits for it to return. Writes results to a new PSUADE file. (To avoid overwriting the original) If a sessionId is supplied with -s, it is assumed the create and submit process has already happened, so it simply blocks waiting for results.
"""
op = optparse.OptionParser(usage="USAGE: %prog PSUDADE_IN PSUADE_OUT CONFIG_FILE",
description=main_do_psuade.__doc__)
add_options(op)
(options, args) = op.parse_args()
assert(len(args) >= 3)
psuadeInFilename = args[0]
psuadeOutFilename = args[1]
configFilename = args[2]
configFile = _open_config(configFilename)
sessionid = options.session
if(sessionid == None):
sessionid = psuade_launch(configFile, psuadeInFilename)
print("SessionID:", sessionid)
# loop checking that the jobs haven't finished yet
jobsLeft = 1
while(jobsLeft > 0):
jobsLeft = jobs_unfinished(configFile, sessionid)
print(jobsLeft)
if(jobsLeft > 0):
time.sleep(jobsLeft*5)
# Now that the jobs are complete, get them all
write_jobs_to_psuade(configFile, sessionid,
psuadeInFilename, psuadeOutFilename)
print("Jobs written to", psuadeOutFilename)
def main_session_stats(args=None):
"""session resource utility, prints basic session statistics
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_session_stats.__doc__)
add_options(op)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception as ex:
op.error(ex)
pages = get_paging(configFile, SECTION, options, subresource=sessionid)
data = load_pages_json(pages)
basic_session_stats(sys.stdout, data, verbose=options.verbose)
# if options.verbose:
# for i in success_l:
# print i['Output']
return data
def main_write_jobs_to_psuade(args=None):
""" Gets all jobs from the session and writes them to a PSUADE file """
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID PSUADE_IN PSUADE_OUT CONFIG_FILE",
description=main_write_jobs_to_psuade.__doc__)
add_options(op)
(options, args) = op.parse_args()
assert(len(args) >= 4)
sessionid = args[0]
psuadeInFilename = args[1]
psuadeOutFilename = args[2]
configFile = _open_config(args[3])
if(not configFile.has_section(SECTION) or
not configFile.has_option(SECTION, "url") or
configFile.get(SECTION, "url") == ""):
# Local Psuade based run
shutil.copyfile("psuadeData", args[2])
else:
# Gateway run
write_jobs_to_psuade(configFile, sessionid,
psuadeInFilename, psuadeOutFilename, options)
print("Jobs written to", args[2])
def main_list(args=None, func=_print_simulation_list):
"""Retrieves list of all simulations, by default prints in human readable format.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main_list.__doc__)
add_options(op)
add_json_option(op)
(options, args) = op.parse_args(args)
if len(args) != 1:
op.error('expecting 1 argument')
if options.json:
func = _print_json
configFile = _open_config(args[0])
query = {}
if options.verbose:
query['verbose'] = options.verbose
options.page = 1
content = get_page(configFile, SECTION, **query)
data = load_pages_json([content])
if func:
func(data, options.verbose)
return data
def main(args=None, func=_print_as_json):
"""Queries for job resources based on select criteria, by default prints JSON array of jobs.
"""
global states
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main.__doc__)
op.add_option("-j", "--jobid",
action="store", dest="subresource", default=None,
help="JOB ID")
op.add_option("-n", "--sim",
action="store", dest="simulation", default=None,
help="Simulation Name")
op.add_option("-x", "--state",
action="store", dest="state", default=None,
help="Job Status to query: %s" %list(states))
op.add_option("-c", "--consumer",
action="store", dest="consumer", default=None,
help="Consumer GUID to query")
op.add_option("-b", "--basic",
action="store_true", dest="basic",
help="Print Basic Information About Job(s)")
add_options(op)
add_session_option(op)
(options, args) = op.parse_args(args)
configFile = _open_config(args[0])
query = {}
if options.session: query['session'] = options.session
if options.simulation: query['simulation'] = options.simulation
if options.subresource: query['subresource'] = options.subresource
if options.state: query['state'] = options.state
if options.consumer: query['consumer'] = options.consumer
if options.verbose: query['verbose'] = options.verbose
if options.subresource:
page = get_page(configFile, SECTION, **query)
job = json.loads(page)
if options.basic:
write_basic_job_info(sys.stdout, job, verbose=options.verbose)
elif func is not None and callable(func):
func(job, sys.stdout)
return job
try:
pages = get_paging(configFile, SECTION, options, **query)
except HTTPError, ex:
print ex
print ex.readlines()
return
def main_list(args=None, func=_print_numbered_lines):
"""Prints human readable listing of all session GUIDs.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main_list.__doc__)
add_options(op)
add_json_option(op)
(options, args) = op.parse_args(args)
try:
configFile = _open_config(*args)
except Exception, ex:
op.error(ex.Message)
def main_session_stats(args=None):
"""session resource utility, prints basic session statistics
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_session_stats.__doc__)
add_options(op)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception, ex:
op.error(ex.Message)
def main_jobs_status(args=None, func=_print_as_json):
"""session resource utility, lists all session resources
"""
op = optparse.OptionParser(
usage="USAGE: %prog [options] SESSIONID CONFIG_FILE",
description=main_jobs_status.__doc__)
add_options(op)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 argument')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception, ex:
op.error(ex.Message)
def main_get_results(args=None, func=_print_page):
"""Gets the results of all the completed jobs in this session
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_get_results.__doc__)
add_options(op)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
cp = _open_config(*args[1:])
except Exception, ex:
op.error(ex.Message)
def main_session_graphs(args=None):
"""session resource utility, creates session stats graphs with R
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_session_stats.__doc__)
add_options(op)
op.add_option("--plot",
dest="plot",
help="Plot type (see --list). "
"Any unique prefix is OK. (default=%default)",
default="density")
op.add_option("--list",
dest="do_list",
action="store_true",
help="List plot types and metrics.")
op.add_option("--metric",
dest="opt_metric",
help="Metric to show in plot (see --list).")
(options, args) = op.parse_args(args)
# If --list, list plots and stop
if options.do_list:
print("Plot types:")
for pt in sorted(Plot.TYPES):
print(" {type:20s} {desc}".format(type=pt, desc=Plot.TYPES[pt]))
print("Plot metrics:")
for pm in Plot.METRICS:
print(" {m:20s}".format(m=pm))
sys.exit(0)
if options.opt_metric is not None and \
options.opt_metric not in Plot.METRICS:
op.error("Bad metric {0}, must be in: {1}".format(
options.opt_metric, ', '.join(Plot.METRICS)))
# Import R stuff
try:
import rpy2
except ImportError, err:
op.error("rpy2 and a working R installation are required.\n"
"Import error: {0}".format(err))
def main_list(args=None, func=_print_numbered_lines):
"""Prints human readable listing of all session GUIDs.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main_list.__doc__)
add_options(op)
add_json_option(op)
(options, args) = op.parse_args(args)
try:
configFile = _open_config(*args)
except Exception as ex:
op.error(ex)
if options.json:
func = _print_as_json
query = {}
pages = get_paging(configFile, SECTION, options, **query)
data = load_pages_json(pages)
if func:
func(data)
return data
def main(args=None, func=_print_as_json):
"""Queries for job resources based on select criteria, by default prints JSON array of jobs.
"""
global states
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main.__doc__)
op.add_option("-j",
"--jobid",
action="store",
dest="subresource",
default=None,
help="JOB ID")
op.add_option("-n",
"--sim",
action="store",
dest="simulation",
default=None,
help="Simulation Name")
op.add_option("-x",
"--state",
action="store",
dest="state",
default=None,
help="Job Status to query: %s" % list(states))
op.add_option("-c",
"--consumer",
action="store",
dest="consumer",
default=None,
help="Consumer GUID to query")
op.add_option("-b",
"--basic",
action="store_true",
dest="basic",
help="Print Basic Information About Job(s)")
add_options(op)
add_session_option(op)
(options, args) = op.parse_args(args)
configFile = _open_config(args[0])
query = {}
if options.session:
query['session'] = options.session
if options.simulation:
query['simulation'] = options.simulation
if options.subresource:
query['subresource'] = options.subresource
if options.state:
query['state'] = options.state
if options.consumer:
query['consumer'] = options.consumer
if options.verbose:
query['verbose'] = options.verbose
if options.subresource:
page = get_page(configFile, SECTION, **query)
job = json.loads(page)
if options.basic:
write_basic_job_info(sys.stdout, job, verbose=options.verbose)
elif func is not None and callable(func):
func(job, sys.stdout)
return job
try:
pages = get_paging(configFile, SECTION, options, **query)
except HTTPError as ex:
print(ex)
print(ex.readlines())
return
#states = set(['submit', 'create', 'setup', 'running', 'success', 'warning', 'error', 'expired', 'cancel', 'terminate'])
all = load_pages_json(pages)
fstates = set(map(lambda e: e['State'], all))
if not fstates.issubset(states):
_log.debug('**NOTE: Unexpected State Found')
states = states.union(fstates)
if options.basic:
print("Total JOBS: %d" % len(all))
for state in states:
if options.state is not None and options.state != state:
continue
flist = filter(lambda e: e['State'] == state, all)
print("\t%s JOBS: %d" % (state.upper(), len(flist)))
if options.verbose:
for i in flist:
print
for k, v in i.items():
print("\t%12s -- %s" % (k, v))
else:
print("\t%s" % map(lambda m: m['Id'], flist))
elif func is not None and callable(func):
_print_as_json(all)
return all
def main_session_graphs(args=None):
"""session resource utility, creates session stats graphs with R
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_session_stats.__doc__)
add_options(op)
op.add_option("--plot",
dest="plot",
help="Plot type (see --list). "
"Any unique prefix is OK. (default=%default)",
default="density")
op.add_option("--list",
dest="do_list",
action="store_true",
help="List plot types and metrics.")
op.add_option("--metric",
dest="opt_metric",
help="Metric to show in plot (see --list).")
(options, args) = op.parse_args(args)
# If --list, list plots and stop
if options.do_list:
print("Plot types:")
for pt in sorted(Plot.TYPES):
print(" {type:20s} {desc}".format(type=pt, desc=Plot.TYPES[pt]))
print("Plot metrics:")
for pm in Plot.METRICS:
print(" {m:20s}".format(m=pm))
sys.exit(0)
if options.opt_metric is not None and \
options.opt_metric not in Plot.METRICS:
op.error("Bad metric {0}, must be in: {1}".format(
options.opt_metric, ', '.join(Plot.METRICS)))
# Import R stuff
try:
import rpy2
except ImportError as err:
op.error("rpy2 and a working R installation are required.\n"
"Import error: {0}".format(err))
from turbine import rpython
# Parse options
if len(args) != 2:
op.error('expecting 2 arguments')
if options.plot not in Plot.TYPES: # XXX: accept prefix
op.error('bad plot type {p}, use --list'.format(p=options.plot))
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception as ex:
op.error(ex)
# Fetch data
pages = get_paging(configFile, SECTION, options, subresource=sessionid)
data = load_pages_json(pages)
# Create R data frame
tbl, hdr = session_perf_from_json(data)
coltypes = rpython.get_coltypes([column[0] for column in tbl])
df = rpython.make_data_frame(hdr, coltypes, cols=tbl)
# Create and draw selected plot
plot_opt = {}
for k in dir(options):
if k.startswith("opt_"):
plot_opt[k[4:]] = getattr(options, k)
plot = Plot(df, hdr, **plot_opt)
device = "png"
filename = "%s_%s.%s" % (options.plot, sessionid, device)
getattr(plot, "plot_" + options.plot)(filename, device=device)
return {options.plot: filename}