def main_session_stats(args=None):
"""session resource utility, prints basic session statistics
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_session_stats.__doc__)
add_options(op)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception as ex:
op.error(ex)
pages = get_paging(configFile, SECTION, options, subresource=sessionid)
data = load_pages_json(pages)
basic_session_stats(sys.stdout, data, verbose=options.verbose)
# if options.verbose:
# for i in success_l:
# print i['Output']
return data
def main_list(args=None, func=_print_simulation_list):
"""Retrieves list of all simulations, by default prints in human readable format.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main_list.__doc__)
add_options(op)
add_json_option(op)
(options, args) = op.parse_args(args)
if len(args) != 1:
op.error('expecting 1 argument')
if options.json:
func = _print_json
configFile = _open_config(args[0])
query = {}
if options.verbose:
query['verbose'] = options.verbose
options.page = 1
content = get_page(configFile, SECTION, **query)
data = load_pages_json([content])
if func:
func(data, options.verbose)
return data
def get_results(cp, sessionid, options):
""" Gets all jobs from the session and return them as a list """
# default values for a few important items
url = '/'.join([cp.get(SECTION, 'url').strip('/'), sessionid])
#rpp = getFromConfigWithDefaults(cp,SECTION, 'rpp', '100')
verbose = getFromConfigWithDefaults(cp, SECTION, 'verbose', 'false')
pagenum = options.page
rpp = options.rpp
query = {}
query['subresource'] = sessionid
query['verbose'] = verbose
query['rpp'] = str(rpp)
query['page'] = pagenum
_log.getLogger(__name__).debug('query: %s' % (query))
pages = []
_log.getLogger(__name__).debug("downloading results 1-%d" % (int(rpp)))
if (pagenum > 0):
thispage = get_paging_by_url(url, cp, SECTION, query)
pages.extend(thispage)
else:
pagenum = 1
query['page'] = 1
thispage = get_paging_by_url(url, cp, SECTION, query)
while (thispage[0] != '[]'):
pages.extend(thispage)
pagenum += 1
query['page'] = pagenum
_log.getLogger(__name__).debug(
"downloading results %d-%d" %
((pagenum - 1) * int(rpp), pagenum * int(rpp)))
thispage = get_paging_by_url(url, cp, SECTION, query)
#assert(type(pages), list)
print("all results recieved.")
data = load_pages_json(pages)
return data
def main_list(args=None, func=_print_numbered_lines):
"""Prints human readable listing of all session GUIDs.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main_list.__doc__)
add_options(op)
add_json_option(op)
(options, args) = op.parse_args(args)
try:
configFile = _open_config(*args)
except Exception as ex:
op.error(ex)
if options.json:
func = _print_as_json
query = {}
pages = get_paging(configFile, SECTION, options, **query)
data = load_pages_json(pages)
if func:
func(data)
return data
def main(args=None, func=_print_as_json):
"""Queries for job resources based on select criteria, by default prints JSON array of jobs.
"""
global states
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main.__doc__)
op.add_option("-j",
"--jobid",
action="store",
dest="subresource",
default=None,
help="JOB ID")
op.add_option("-n",
"--sim",
action="store",
dest="simulation",
default=None,
help="Simulation Name")
op.add_option("-x",
"--state",
action="store",
dest="state",
default=None,
help="Job Status to query: %s" % list(states))
op.add_option("-c",
"--consumer",
action="store",
dest="consumer",
default=None,
help="Consumer GUID to query")
op.add_option("-b",
"--basic",
action="store_true",
dest="basic",
help="Print Basic Information About Job(s)")
add_options(op)
add_session_option(op)
(options, args) = op.parse_args(args)
configFile = _open_config(args[0])
query = {}
if options.session:
query['session'] = options.session
if options.simulation:
query['simulation'] = options.simulation
if options.subresource:
query['subresource'] = options.subresource
if options.state:
query['state'] = options.state
if options.consumer:
query['consumer'] = options.consumer
if options.verbose:
query['verbose'] = options.verbose
if options.subresource:
page = get_page(configFile, SECTION, **query)
job = json.loads(page)
if options.basic:
write_basic_job_info(sys.stdout, job, verbose=options.verbose)
elif func is not None and callable(func):
func(job, sys.stdout)
return job
try:
pages = get_paging(configFile, SECTION, options, **query)
except HTTPError as ex:
print(ex)
print(ex.readlines())
return
#states = set(['submit', 'create', 'setup', 'running', 'success', 'warning', 'error', 'expired', 'cancel', 'terminate'])
all = load_pages_json(pages)
fstates = set(map(lambda e: e['State'], all))
if not fstates.issubset(states):
_log.debug('**NOTE: Unexpected State Found')
states = states.union(fstates)
if options.basic:
print("Total JOBS: %d" % len(all))
for state in states:
if options.state is not None and options.state != state:
continue
flist = filter(lambda e: e['State'] == state, all)
print("\t%s JOBS: %d" % (state.upper(), len(flist)))
if options.verbose:
for i in flist:
print
for k, v in i.items():
print("\t%12s -- %s" % (k, v))
else:
print("\t%s" % map(lambda m: m['Id'], flist))
elif func is not None and callable(func):
_print_as_json(all)
return all
def main_session_graphs(args=None):
"""session resource utility, creates session stats graphs with R
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_session_stats.__doc__)
add_options(op)
op.add_option("--plot",
dest="plot",
help="Plot type (see --list). "
"Any unique prefix is OK. (default=%default)",
default="density")
op.add_option("--list",
dest="do_list",
action="store_true",
help="List plot types and metrics.")
op.add_option("--metric",
dest="opt_metric",
help="Metric to show in plot (see --list).")
(options, args) = op.parse_args(args)
# If --list, list plots and stop
if options.do_list:
print("Plot types:")
for pt in sorted(Plot.TYPES):
print(" {type:20s} {desc}".format(type=pt, desc=Plot.TYPES[pt]))
print("Plot metrics:")
for pm in Plot.METRICS:
print(" {m:20s}".format(m=pm))
sys.exit(0)
if options.opt_metric is not None and \
options.opt_metric not in Plot.METRICS:
op.error("Bad metric {0}, must be in: {1}".format(
options.opt_metric, ', '.join(Plot.METRICS)))
# Import R stuff
try:
import rpy2
except ImportError as err:
op.error("rpy2 and a working R installation are required.\n"
"Import error: {0}".format(err))
from turbine import rpython
# Parse options
if len(args) != 2:
op.error('expecting 2 arguments')
if options.plot not in Plot.TYPES: # XXX: accept prefix
op.error('bad plot type {p}, use --list'.format(p=options.plot))
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception as ex:
op.error(ex)
# Fetch data
pages = get_paging(configFile, SECTION, options, subresource=sessionid)
data = load_pages_json(pages)
# Create R data frame
tbl, hdr = session_perf_from_json(data)
coltypes = rpython.get_coltypes([column[0] for column in tbl])
df = rpython.make_data_frame(hdr, coltypes, cols=tbl)
# Create and draw selected plot
plot_opt = {}
for k in dir(options):
if k.startswith("opt_"):
plot_opt[k[4:]] = getattr(options, k)
plot = Plot(df, hdr, **plot_opt)
device = "png"
filename = "%s_%s.%s" % (options.plot, sessionid, device)
getattr(plot, "plot_" + options.plot)(filename, device=device)
return {options.plot: filename}
if options.basic:
write_basic_job_info(sys.stdout, job, verbose=options.verbose)
elif func is not None and callable(func):
func(job, sys.stdout)
return job
try:
pages = get_paging(configFile, SECTION, options, **query)
except HTTPError, ex:
print ex
print ex.readlines()
return
#states = set(['submit', 'create', 'setup', 'running', 'success', 'warning', 'error', 'expired', 'cancel', 'terminate'])
all = load_pages_json(pages)
fstates = set(map(lambda e: e['State'], all))
if not fstates.issubset(states):
_log.debug('**NOTE: Unexpected State Found')
states = states.union(fstates)
if options.basic:
print "Total JOBS: %d" %len(all)
for state in states:
if options.state is not None and options.state != state: continue
flist = filter(lambda e: e['State'] == state, all)
print "\t%s JOBS: %d" %(state.upper(), len(flist))
if options.verbose:
for i in flist:
print
for k,v in i.items():
description=main_list.__doc__)
add_options(op)
add_json_option(op)
(options, args) = op.parse_args(args)
try:
configFile = _open_config(*args)
except Exception, ex:
op.error(ex.Message)
if options.json:
func = _print_as_json
query = {}
pages = get_paging(configFile, SECTION, options, **query)
data = load_pages_json(pages)
if func:
func(data)
return data
def jobs_status(configFile, sessionid):
query = {}
query['subresource'] = sessionid + "/status"
data = get_page(configFile, SECTION, **query)
return json.loads(data)
def main_jobs_status(args=None, func=_print_as_json):
"""session resource utility, lists all session resources