def main_do_psuade(args=None): """Takes a PSUADE file and submits all the jobs contained therein as a new session. Waits for it to return. Writes results to a new PSUADE file. (To avoid overwriting the original) If a sessionId is supplied with -s, it is assumed the create and submit process has already happened, so it simply blocks waiting for results. """ op = optparse.OptionParser(usage="USAGE: %prog PSUDADE_IN PSUADE_OUT CONFIG_FILE", description=main_do_psuade.__doc__) add_options(op) (options, args) = op.parse_args() assert(len(args) >= 3) psuadeInFilename = args[0] psuadeOutFilename = args[1] configFilename = args[2] configFile = _open_config(configFilename) sessionid = options.session if(sessionid == None): sessionid = psuade_launch(configFile, psuadeInFilename) print("SessionID:", sessionid) # loop checking that the jobs haven't finished yet jobsLeft = 1 while(jobsLeft > 0): jobsLeft = jobs_unfinished(configFile, sessionid) print(jobsLeft) if(jobsLeft > 0): time.sleep(jobsLeft*5) # Now that the jobs are complete, get them all write_jobs_to_psuade(configFile, sessionid, psuadeInFilename, psuadeOutFilename) print("Jobs written to", psuadeOutFilename)
def main_session_stats(args=None): """session resource utility, prints basic session statistics """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_session_stats.__doc__) add_options(op) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception as ex: op.error(ex) pages = get_paging(configFile, SECTION, options, subresource=sessionid) data = load_pages_json(pages) basic_session_stats(sys.stdout, data, verbose=options.verbose) # if options.verbose: # for i in success_l: # print i['Output'] return data
def main_write_jobs_to_psuade(args=None): """ Gets all jobs from the session and writes them to a PSUADE file """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID PSUADE_IN PSUADE_OUT CONFIG_FILE", description=main_write_jobs_to_psuade.__doc__) add_options(op) (options, args) = op.parse_args() assert(len(args) >= 4) sessionid = args[0] psuadeInFilename = args[1] psuadeOutFilename = args[2] configFile = _open_config(args[3]) if(not configFile.has_section(SECTION) or not configFile.has_option(SECTION, "url") or configFile.get(SECTION, "url") == ""): # Local Psuade based run shutil.copyfile("psuadeData", args[2]) else: # Gateway run write_jobs_to_psuade(configFile, sessionid, psuadeInFilename, psuadeOutFilename, options) print("Jobs written to", args[2])
def main_list(args=None, func=_print_simulation_list): """Retrieves list of all simulations, by default prints in human readable format. """ op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main_list.__doc__) add_options(op) add_json_option(op) (options, args) = op.parse_args(args) if len(args) != 1: op.error('expecting 1 argument') if options.json: func = _print_json configFile = _open_config(args[0]) query = {} if options.verbose: query['verbose'] = options.verbose options.page = 1 content = get_page(configFile, SECTION, **query) data = load_pages_json([content]) if func: func(data, options.verbose) return data
def main(args=None, func=_print_as_json): """Queries for job resources based on select criteria, by default prints JSON array of jobs. """ global states op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main.__doc__) op.add_option("-j", "--jobid", action="store", dest="subresource", default=None, help="JOB ID") op.add_option("-n", "--sim", action="store", dest="simulation", default=None, help="Simulation Name") op.add_option("-x", "--state", action="store", dest="state", default=None, help="Job Status to query: %s" %list(states)) op.add_option("-c", "--consumer", action="store", dest="consumer", default=None, help="Consumer GUID to query") op.add_option("-b", "--basic", action="store_true", dest="basic", help="Print Basic Information About Job(s)") add_options(op) add_session_option(op) (options, args) = op.parse_args(args) configFile = _open_config(args[0]) query = {} if options.session: query['session'] = options.session if options.simulation: query['simulation'] = options.simulation if options.subresource: query['subresource'] = options.subresource if options.state: query['state'] = options.state if options.consumer: query['consumer'] = options.consumer if options.verbose: query['verbose'] = options.verbose if options.subresource: page = get_page(configFile, SECTION, **query) job = json.loads(page) if options.basic: write_basic_job_info(sys.stdout, job, verbose=options.verbose) elif func is not None and callable(func): func(job, sys.stdout) return job try: pages = get_paging(configFile, SECTION, options, **query) except HTTPError, ex: print ex print ex.readlines() return
def main_list(args=None, func=_print_numbered_lines): """Prints human readable listing of all session GUIDs. """ op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main_list.__doc__) add_options(op) add_json_option(op) (options, args) = op.parse_args(args) try: configFile = _open_config(*args) except Exception, ex: op.error(ex.Message)
def main_session_stats(args=None): """session resource utility, prints basic session statistics """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_session_stats.__doc__) add_options(op) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception, ex: op.error(ex.Message)
def main_jobs_status(args=None, func=_print_as_json): """session resource utility, lists all session resources """ op = optparse.OptionParser( usage="USAGE: %prog [options] SESSIONID CONFIG_FILE", description=main_jobs_status.__doc__) add_options(op) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 argument') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception, ex: op.error(ex.Message)
def main_get_results(args=None, func=_print_page): """Gets the results of all the completed jobs in this session """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_get_results.__doc__) add_options(op) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: cp = _open_config(*args[1:]) except Exception, ex: op.error(ex.Message)
def main_session_graphs(args=None): """session resource utility, creates session stats graphs with R """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_session_stats.__doc__) add_options(op) op.add_option("--plot", dest="plot", help="Plot type (see --list). " "Any unique prefix is OK. (default=%default)", default="density") op.add_option("--list", dest="do_list", action="store_true", help="List plot types and metrics.") op.add_option("--metric", dest="opt_metric", help="Metric to show in plot (see --list).") (options, args) = op.parse_args(args) # If --list, list plots and stop if options.do_list: print("Plot types:") for pt in sorted(Plot.TYPES): print(" {type:20s} {desc}".format(type=pt, desc=Plot.TYPES[pt])) print("Plot metrics:") for pm in Plot.METRICS: print(" {m:20s}".format(m=pm)) sys.exit(0) if options.opt_metric is not None and \ options.opt_metric not in Plot.METRICS: op.error("Bad metric {0}, must be in: {1}".format( options.opt_metric, ', '.join(Plot.METRICS))) # Import R stuff try: import rpy2 except ImportError, err: op.error("rpy2 and a working R installation are required.\n" "Import error: {0}".format(err))
def main_list(args=None, func=_print_numbered_lines): """Prints human readable listing of all session GUIDs. """ op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main_list.__doc__) add_options(op) add_json_option(op) (options, args) = op.parse_args(args) try: configFile = _open_config(*args) except Exception as ex: op.error(ex) if options.json: func = _print_as_json query = {} pages = get_paging(configFile, SECTION, options, **query) data = load_pages_json(pages) if func: func(data) return data
def main(args=None, func=_print_as_json): """Queries for job resources based on select criteria, by default prints JSON array of jobs. """ global states op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main.__doc__) op.add_option("-j", "--jobid", action="store", dest="subresource", default=None, help="JOB ID") op.add_option("-n", "--sim", action="store", dest="simulation", default=None, help="Simulation Name") op.add_option("-x", "--state", action="store", dest="state", default=None, help="Job Status to query: %s" % list(states)) op.add_option("-c", "--consumer", action="store", dest="consumer", default=None, help="Consumer GUID to query") op.add_option("-b", "--basic", action="store_true", dest="basic", help="Print Basic Information About Job(s)") add_options(op) add_session_option(op) (options, args) = op.parse_args(args) configFile = _open_config(args[0]) query = {} if options.session: query['session'] = options.session if options.simulation: query['simulation'] = options.simulation if options.subresource: query['subresource'] = options.subresource if options.state: query['state'] = options.state if options.consumer: query['consumer'] = options.consumer if options.verbose: query['verbose'] = options.verbose if options.subresource: page = get_page(configFile, SECTION, **query) job = json.loads(page) if options.basic: write_basic_job_info(sys.stdout, job, verbose=options.verbose) elif func is not None and callable(func): func(job, sys.stdout) return job try: pages = get_paging(configFile, SECTION, options, **query) except HTTPError as ex: print(ex) print(ex.readlines()) return #states = set(['submit', 'create', 'setup', 'running', 'success', 'warning', 'error', 'expired', 'cancel', 'terminate']) all = load_pages_json(pages) fstates = set(map(lambda e: e['State'], all)) if not fstates.issubset(states): _log.debug('**NOTE: Unexpected State Found') states = states.union(fstates) if options.basic: print("Total JOBS: %d" % len(all)) for state in states: if options.state is not None and options.state != state: continue flist = filter(lambda e: e['State'] == state, all) print("\t%s JOBS: %d" % (state.upper(), len(flist))) if options.verbose: for i in flist: print for k, v in i.items(): print("\t%12s -- %s" % (k, v)) else: print("\t%s" % map(lambda m: m['Id'], flist)) elif func is not None and callable(func): _print_as_json(all) return all
def main_session_graphs(args=None): """session resource utility, creates session stats graphs with R """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_session_stats.__doc__) add_options(op) op.add_option("--plot", dest="plot", help="Plot type (see --list). " "Any unique prefix is OK. (default=%default)", default="density") op.add_option("--list", dest="do_list", action="store_true", help="List plot types and metrics.") op.add_option("--metric", dest="opt_metric", help="Metric to show in plot (see --list).") (options, args) = op.parse_args(args) # If --list, list plots and stop if options.do_list: print("Plot types:") for pt in sorted(Plot.TYPES): print(" {type:20s} {desc}".format(type=pt, desc=Plot.TYPES[pt])) print("Plot metrics:") for pm in Plot.METRICS: print(" {m:20s}".format(m=pm)) sys.exit(0) if options.opt_metric is not None and \ options.opt_metric not in Plot.METRICS: op.error("Bad metric {0}, must be in: {1}".format( options.opt_metric, ', '.join(Plot.METRICS))) # Import R stuff try: import rpy2 except ImportError as err: op.error("rpy2 and a working R installation are required.\n" "Import error: {0}".format(err)) from turbine import rpython # Parse options if len(args) != 2: op.error('expecting 2 arguments') if options.plot not in Plot.TYPES: # XXX: accept prefix op.error('bad plot type {p}, use --list'.format(p=options.plot)) sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception as ex: op.error(ex) # Fetch data pages = get_paging(configFile, SECTION, options, subresource=sessionid) data = load_pages_json(pages) # Create R data frame tbl, hdr = session_perf_from_json(data) coltypes = rpython.get_coltypes([column[0] for column in tbl]) df = rpython.make_data_frame(hdr, coltypes, cols=tbl) # Create and draw selected plot plot_opt = {} for k in dir(options): if k.startswith("opt_"): plot_opt[k[4:]] = getattr(options, k) plot = Plot(df, hdr, **plot_opt) device = "png" filename = "%s_%s.%s" % (options.plot, sessionid, device) getattr(plot, "plot_" + options.plot)(filename, device=device) return {options.plot: filename}