def main_do_psuade(args=None): """Takes a PSUADE file and submits all the jobs contained therein as a new session. Waits for it to return. Writes results to a new PSUADE file. (To avoid overwriting the original) If a sessionId is supplied with -s, it is assumed the create and submit process has already happened, so it simply blocks waiting for results. """ op = optparse.OptionParser(usage="USAGE: %prog PSUDADE_IN PSUADE_OUT CONFIG_FILE", description=main_do_psuade.__doc__) add_options(op) (options, args) = op.parse_args() assert(len(args) >= 3) psuadeInFilename = args[0] psuadeOutFilename = args[1] configFilename = args[2] configFile = _open_config(configFilename) sessionid = options.session if(sessionid == None): sessionid = psuade_launch(configFile, psuadeInFilename) print("SessionID:", sessionid) # loop checking that the jobs haven't finished yet jobsLeft = 1 while(jobsLeft > 0): jobsLeft = jobs_unfinished(configFile, sessionid) print(jobsLeft) if(jobsLeft > 0): time.sleep(jobsLeft*5) # Now that the jobs are complete, get them all write_jobs_to_psuade(configFile, sessionid, psuadeInFilename, psuadeOutFilename) print("Jobs written to", psuadeOutFilename)
def main_psuade_jobs_unfinished(args=None): """Reports how many jobs remain unfinished in the session (Useful for UQ GUI) """ op = optparse.OptionParser(usage="USAGE: %prog [options] SESSIONID CONFIG_FILE", description=main_psuade_jobs_unfinished.__doc__) (options, args) = op.parse_args(args) if len(args) != 2: op.error('expecting 2 arguments') sessionid = args[0] configFile = _open_config(args[1]) numunfinished = 0 if(not configFile.has_section(SECTION) or not configFile.has_option(SECTION, "url")): # Local Psuade based run numunfinished = local_unfinished(configFile, "psuadeData") else: # Gateway run numunfinished = jobs_unfinished(configFile, sessionid) if(numunfinished < 0): numunfinished = 0 print print("{unfinished:", numunfinished, "}")
def main_update_config_instanceType(args=None, func=_print_page): """Sets the AWS EC2 instance type for new virtual machines. This feature is only relevant for the AWS EC2 deployment. By default prints entire resultant configuration in JSON. """ op = optparse.OptionParser( usage= "USAGE: %prog [options] [t1.micro | m1.small | c1.medium] CONFIG_FILE", description=main_update_config_instanceType.__doc__) # add_options(op) op.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output") (options, args) = op.parse_args(args) if len(args) != 2: op.error('expecting 2 argument') configFile = _open_config(args[1]) query = dict(subresource='/config') page = put_page(configFile, SECTION, json.dumps(dict(instance=args[0])), **query) if func: func(page) return page
def main_write_jobs_to_psuade(args=None): """ Gets all jobs from the session and writes them to a PSUADE file """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID PSUADE_IN PSUADE_OUT CONFIG_FILE", description=main_write_jobs_to_psuade.__doc__) add_options(op) (options, args) = op.parse_args() assert(len(args) >= 4) sessionid = args[0] psuadeInFilename = args[1] psuadeOutFilename = args[2] configFile = _open_config(args[3]) if(not configFile.has_section(SECTION) or not configFile.has_option(SECTION, "url") or configFile.get(SECTION, "url") == ""): # Local Psuade based run shutil.copyfile("psuadeData", args[2]) else: # Gateway run write_jobs_to_psuade(configFile, sessionid, psuadeInFilename, psuadeOutFilename, options) print("Jobs written to", args[2])
def main_list(args=None, func=_print_simulation_list): """Retrieves list of all simulations, by default prints in human readable format. """ op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main_list.__doc__) add_options(op) add_json_option(op) (options, args) = op.parse_args(args) if len(args) != 1: op.error('expecting 1 argument') if options.json: func = _print_json configFile = _open_config(args[0]) query = {} if options.verbose: query['verbose'] = options.verbose options.page = 1 content = get_page(configFile, SECTION, **query) data = load_pages_json([content]) if func: func(data, options.verbose) return data
def main_delete(args=None, func=_print_page): """Delete simulation """ op = optparse.OptionParser( usage="USAGE: %prog SIMULATION_NAME CONFIG_FILE", description=main_delete.__doc__) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') simulationName = args[0] try: configFile = _open_config(*args[1:]) except Exception as ex: op.error(ex) log = _log.getLogger(__name__) try: page = delete_page(configFile, SECTION, subresource='%s' % simulationName) except HTTPError as ex: log.error(ex) raise log.debug("PAGE: %s" % page) return page
def main_update_config_floor(args=None, func=_print_page): """Sets a floor for the number of Consumer processes that should remain running for a interval determined by the server. Currently this is only supported in the AWS EC2 deployment. By default prints entire resultant configuration in JSON. """ op = optparse.OptionParser(usage="USAGE: %prog [options] INT CONFIG_FILE", description=main_update_config_floor.__doc__) # add_options(op) op.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output") (options, args) = op.parse_args(args) if len(args) != 2: op.error('expecting 2 argument') configFile = _open_config(args[1]) query = dict(subresource='/config') page = put_page(configFile, SECTION, json.dumps(dict(floor=int(args[0]))), **query) if func: func(page) return page
def main_get_config(args=None, func=_print_page): """Return configuration settings for top-level Consumer resource, by default print as JSON. These settings are utilized by an orchestrator process (deployment specific). The AWS EC2 orchestator handles auto-scaling of instances. """ op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main_get_config.__doc__) # add_options(op) op.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output") (options, args) = op.parse_args(args) if len(args) != 1: op.error('expecting 1 argument') configFile = _open_config(args[0]) query = dict(subresource='/config') page = get_page(configFile, SECTION, **query) if func: func(page) return page
def main_delete(args=None, func=_print_page): """Delete session and all jobs it contains, prints number of jobs deleted as result of session delete. """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_delete.__doc__) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception as ex: op.error(ex) log = _log.getLogger(__name__) try: page = delete_page(configFile, SECTION, subresource='%s' % sessionid) except HTTPError as ex: log.error(ex) raise log.debug("PAGE: %s" % page) return int(page)
def main_session_stats(args=None): """session resource utility, prints basic session statistics """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_session_stats.__doc__) add_options(op) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception as ex: op.error(ex) pages = get_paging(configFile, SECTION, options, subresource=sessionid) data = load_pages_json(pages) basic_session_stats(sys.stdout, data, verbose=options.verbose) # if options.verbose: # for i in success_l: # print i['Output'] return data
def main_get_session_result_page(args=None, func=_print_page): """Terminate jobs in session in state setup, running. Print number of jobs terminated. """ op = optparse.OptionParser( usage="USAGE: %prog SESSIONID PAGE_NUMBER CONFIG_FILE", description=main_kill_jobs.__doc__) (options, args) = op.parse_args(args) if len(args) < 2: op.error('expecting >= 2 arguments') sessionid = args[0] page_number = args[1] try: configFile = _open_config(*args[2:]) except Exception as ex: op.error(ex) page = get_session_page_results(configFile, sessionid, int(page_number)) if page: page = json.loads(page) #data = int(page) if func: func(page) return page
def setUp(self): """ Open up configuration and setup logging before doing anything. """ self.cp = cp = _open_config(self.config_name) _setup_logging(cp) self.log = logging.getLogger( '%s.%s.%s' % (self._module_name, self.__class__.__name__, self._testMethodName))
def main_create(args=None): """Create an empty Simulation Resource """ op = optparse.OptionParser( usage= "USAGE: %prog [options] SIMULATION_NAME|GUID APPLICATION_NAME CONFIG_FILE", description=main_create.__doc__) op.add_option( "-s", "--simulation", action="store", dest="simulation_name", default=None, help= "override versioning system and specify the GUID identifier for a simulation " ) (options, args) = op.parse_args(args) if len(args) != 3: op.error('expecting 3 arguments') cp = _open_config(args[2]) application = args[1] subresource = args[0] simulation_name = None if options.simulation_name: uuid.UUID(subresource) simulation_name = options.simulation_name else: try: uuid.UUID(subresource) except ValueError: pass else: op.error('must use -s option when specifying a guid, see help') simulation_name = subresource kw = dict(subresource=subresource) data = json.dumps( dict(Application=application, StagedInputs=[], Name=simulation_name)) try: data = put_page(cp, SECTION, data, content_type='application/json', **kw) except HTTPError as ex: _log.error(ex) if hasattr(ex, 'readlines'): _log.error("".join(ex.readlines())) elif hasattr(ex, 'read'): _log.error("".join(ex.read())) raise return json.loads(data)
def main(args=None, func=_print_as_json): """Queries for job resources based on select criteria, by default prints JSON array of jobs. """ global states op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main.__doc__) op.add_option("-j", "--jobid", action="store", dest="subresource", default=None, help="JOB ID") op.add_option("-n", "--sim", action="store", dest="simulation", default=None, help="Simulation Name") op.add_option("-x", "--state", action="store", dest="state", default=None, help="Job Status to query: %s" %list(states)) op.add_option("-c", "--consumer", action="store", dest="consumer", default=None, help="Consumer GUID to query") op.add_option("-b", "--basic", action="store_true", dest="basic", help="Print Basic Information About Job(s)") add_options(op) add_session_option(op) (options, args) = op.parse_args(args) configFile = _open_config(args[0]) query = {} if options.session: query['session'] = options.session if options.simulation: query['simulation'] = options.simulation if options.subresource: query['subresource'] = options.subresource if options.state: query['state'] = options.state if options.consumer: query['consumer'] = options.consumer if options.verbose: query['verbose'] = options.verbose if options.subresource: page = get_page(configFile, SECTION, **query) job = json.loads(page) if options.basic: write_basic_job_info(sys.stdout, job, verbose=options.verbose) elif func is not None and callable(func): func(job, sys.stdout) return job try: pages = get_paging(configFile, SECTION, options, **query) except HTTPError, ex: print ex print ex.readlines() return
def main_create_session(args=None, func=_print_page): """Creates new session resource and prints GUID of created session. """ op = optparse.OptionParser(usage="USAGE: %prog CONFIG_FILE", description=main_create_session.__doc__) (options, args) = op.parse_args(args) try: configFile = _open_config(*args) except Exception, ex: op.error(ex.Message)
def main_update(args=None): """Update simulation by essentially doing a PUT with the specified file to the resource or optionally sub-resource. """ op = optparse.OptionParser( usage= "USAGE: %prog [options] SIMULATION_NAME|GUID FILE_NAME CONFIG_FILE", description=main_update.__doc__) op.add_option( "-r", "--resource", action="store", dest="resource", help="select the staged input file name (matches Application)") (options, args) = op.parse_args(args) if len(args) != 3: op.error('expecting 3 arguments') log = _log.getLogger('%s.main_update' % __name__) log.debug(args) file_name = args[1] if not os.path.isfile(file_name): op.error('expecting a file for argument 2') if options.resource is None: op.error('require resource option to be specified') configFile = _open_config(args[2]) simulation = args[0] kw = {} out = sys.stdout kw['subresource'] = '%s/input/%s' % (simulation, options.resource) with open(file_name, 'rb') as fd: contents = fd.read() try: data = put_page(configFile, SECTION, contents, **kw) except HTTPError as ex: _log.error("HTTP Code %d : %s", ex.code, ex.msg) if hasattr(ex, 'readlines'): _log.debug("".join(ex.readlines())) else: _log.debug("".join(ex.read())) raise except urllib.error.URLError as ex: _log.error("URLError : %s", ex.reason) raise return data
def main_list(args=None, func=_print_numbered_lines): """Prints human readable listing of all session GUIDs. """ op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main_list.__doc__) add_options(op) add_json_option(op) (options, args) = op.parse_args(args) try: configFile = _open_config(*args) except Exception, ex: op.error(ex.Message)
def main_get_consumer_by_guid(args=None, func=_print_page): """Retrieves consumer by GUID """ op = optparse.OptionParser(usage="USAGE: %prog [options] CONSUMER_GUID CONFIG_FILE", description=main.__doc__) (options, args) = op.parse_args(args) if len(args) != 2: op.error('expecting 2 arguments') configFile = _open_config(args[1]) query = dict(subresource='/%s' %args[0]) page = get_page(configFile, SECTION, **query) data = json.loads(page) if func: func(data) return data
def main_start_jobs(args=None, func=_print_page): """Move all jobs in states pause and create to submit. Prints the number of jobs moved to state submit. """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_start_jobs.__doc__) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception, ex: op.error(ex.Message)
def main_csv_launch(args=None): """Takes a CSV file and submits all the jobs contained therein as a new session. Returns immediatly.""" op = optparse.OptionParser(usage="USAGE: %prog CSV_IN CONFIG_FILE", description=main_csv_launch.__doc__) (options, args) = op.parse_args() assert (len(args) == 2) csvInFilename = args[0] configFilename = args[1] configFile = _open_config(configFilename) print(csv_launch(configFile, csvInFilename))
def main_kill_jobs(args=None, func=_print_page): """Terminate jobs in session in state setup, running. Print number of jobs terminated. """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_kill_jobs.__doc__) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception, ex: op.error(ex.Message)
def main_delete(args=None, func=_print_page): """Delete session and all jobs it contains, prints number of jobs deleted as result of session delete. """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_delete.__doc__) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception, ex: op.error(ex.Message)
def getNumUnfinishedRunSamples(): if not LocalExecutionModule.runStarted: return -1 elif LocalExecutionModule.runComplete: return 0 if not os.path.exists('psuadePath'): return -1 if LocalExecutionModule.runType == Model.LOCAL: print('getting config') config = _open_config(LocalExecutionModule.configFile) else: config = None numUnfinished = turbine_psuade_session_script.local_unfinished( config, 'psuadeData') if numUnfinished == 0: LocalExecutionModule.runComplete = True LocalExecutionModule.stdoutF.close() LocalExecutionModule.stderrF.close() #Clean up files psuadeAppsFiles = os.getcwd() + os.path.sep + 'psuadeApps_ct.*' for f in glob.glob(psuadeAppsFiles): os.remove(f) ## # Check if process is stopped with errors ## elif LocalExecutionModule.process.state() != QtCore.QProcess.Running: ## ##elif LocalExecutionModule.process.poll() is not None: #Process done ## LocalExecutionModule.stdoutF.close() ## LocalExecutionModule.stderrF.close() #Clean up files psuadeAppsFiles = os.getcwd() + os.path.sep + 'psuadeApps_ct.*' for f in glob.glob(psuadeAppsFiles): os.remove(f) ## #Open file and assert error from psuade ## errorOutput = LocalExecutionModule.process.readAllStandardError() ## print errorOutput ## standardOutput = LocalExecutionModule.process.readAllStandardOutput() ## print standardOutput #### outF = open(os.getcwd() + '\\stdout') #### lines = outF.readlines() ## raise RuntimeError(lines) return numUnfinished
def main_log(args=None, func=_print_page): """Retrieves logging messages from compute resource running the specified Consumer. Log messages are printed to screen in order. This functionality is not available in all deployments. """ op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main_log.__doc__) (options, args) = op.parse_args(args) if len(args) != 2: op.error('expecting 2 arguments') configFile = _open_config(args[1]) query = dict(subresource='/%s/log' %args[0]) page = get_page(configFile, SECTION, **query) if func: func(page) return page
def main_session_stats(args=None): """session resource utility, prints basic session statistics """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_session_stats.__doc__) add_options(op) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception, ex: op.error(ex.Message)
def main_delete(args=None, func=_print_page): """Delete simulation """ op = optparse.OptionParser( usage="USAGE: %prog SIMULATION_NAME CONFIG_FILE", description=main_delete.__doc__) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') simulationName = args[0] try: configFile = _open_config(*args[1:]) except Exception, ex: op.error(ex.Message)
def main(args=None, func=_print_numbered_lines): """List all Consumer resources, by default print in human readable format. """ op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE", description=main.__doc__) # add_options(op) op.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output") """ op.add_option("-p", "--page", type="int", action="store", dest="page", default=0, help="page number") op.add_option("-r", "--rpp", type="int", action="store", dest="rpp", default=1000, help="results per page") """ op.add_option("-s", "--status", action="store", dest="status", help="query on status ['up'|'down'|'error']") add_json_option(op) (options, args) = op.parse_args(args) if len(args) != 1: op.error('expecting 1 argument') configFile = _open_config(args[0]) query = {} if options.status: query['status'] = options.status # NOTE RESOURCE NOT SUPPORTING PAGING #pages = get_paging(configFile, SECTION, options, **query) #data = load_pages_json(pages) page = get_page(configFile, SECTION, **query) data = json.loads(page) if options.json: func = _print_as_json if func: func(data) return data
def main_create_jobs(args=None, func=_print_page): """Appends job descriptions to a session, prints the number of jobs added. """ op = optparse.OptionParser( usage="USAGE: %prog SESSIONID JOBS_FILE CONFIG_FILE", description=main_create_jobs.__doc__) (options, args) = op.parse_args(args) if len(args) < 2: op.error('expecting >= 2 arguments') sessionid = args[0] jobs_file = args[1] try: configFile = _open_config(*args[2:]) except Exception as ex: op.error(ex) log = _log.getLogger(__name__) log.debug("main_create_jobs") try: with open(jobs_file) as fd: input_data = json.load(fd) except FileNotFoundError: input_data = json.loads(jobs_file) assert (type(input_data) is list) for o in input_data: thisset = set(o.keys()) assert thisset.issuperset([ "Input", "Simulation" ]), "Each entry requires fields 'Input' and 'Simulation'" try: page = create_jobs(configFile, sessionid, input_data) except HTTPError as ex: log.error(ex) log.error(ex.fp.read()) raise log.debug("PAGE: %s" % page) data = json.loads(page) if (func): func(data) return data
def main_jobs_status(args=None, func=_print_as_json): """session resource utility, lists all session resources """ op = optparse.OptionParser( usage="USAGE: %prog [options] SESSIONID CONFIG_FILE", description=main_jobs_status.__doc__) add_options(op) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 argument') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception, ex: op.error(ex.Message)
def main_stop_jobs(args=None): """Move all jobs in state submit to pause. Prints the number of jobs moved to state pause. """ op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE", description=main_stop_jobs.__doc__) (options, args) = op.parse_args(args) if len(args) < 1: op.error('expecting >= 1 arguments') sessionid = args[0] try: configFile = _open_config(*args[1:]) except Exception as ex: op.error(ex) print(stop_jobs(configFile, sessionid))