def main_do_psuade(args=None):
"""Takes a PSUADE file and submits all the jobs contained therein as a new session. Waits for it to return. Writes results to a new PSUADE file. (To avoid overwriting the original) If a sessionId is supplied with -s, it is assumed the create and submit process has already happened, so it simply blocks waiting for results.
"""
op = optparse.OptionParser(usage="USAGE: %prog PSUDADE_IN PSUADE_OUT CONFIG_FILE",
description=main_do_psuade.__doc__)
add_options(op)
(options, args) = op.parse_args()
assert(len(args) >= 3)
psuadeInFilename = args[0]
psuadeOutFilename = args[1]
configFilename = args[2]
configFile = _open_config(configFilename)
sessionid = options.session
if(sessionid == None):
sessionid = psuade_launch(configFile, psuadeInFilename)
print("SessionID:", sessionid)
# loop checking that the jobs haven't finished yet
jobsLeft = 1
while(jobsLeft > 0):
jobsLeft = jobs_unfinished(configFile, sessionid)
print(jobsLeft)
if(jobsLeft > 0):
time.sleep(jobsLeft*5)
# Now that the jobs are complete, get them all
write_jobs_to_psuade(configFile, sessionid,
psuadeInFilename, psuadeOutFilename)
print("Jobs written to", psuadeOutFilename)
def main_psuade_jobs_unfinished(args=None):
"""Reports how many jobs remain unfinished in the session (Useful for UQ GUI)
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] SESSIONID CONFIG_FILE",
description=main_psuade_jobs_unfinished.__doc__)
(options, args) = op.parse_args(args)
if len(args) != 2:
op.error('expecting 2 arguments')
sessionid = args[0]
configFile = _open_config(args[1])
numunfinished = 0
if(not configFile.has_section(SECTION) or
not configFile.has_option(SECTION, "url")):
# Local Psuade based run
numunfinished = local_unfinished(configFile, "psuadeData")
else:
# Gateway run
numunfinished = jobs_unfinished(configFile, sessionid)
if(numunfinished < 0):
numunfinished = 0
print
print("{unfinished:", numunfinished, "}")
def main_update_config_instanceType(args=None, func=_print_page):
"""Sets the AWS EC2 instance type for new virtual machines. This feature is only
relevant for the AWS EC2 deployment. By default prints entire resultant configuration in JSON.
"""
op = optparse.OptionParser(
usage=
"USAGE: %prog [options] [t1.micro | m1.small | c1.medium] CONFIG_FILE",
description=main_update_config_instanceType.__doc__)
# add_options(op)
op.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
help="verbose output")
(options, args) = op.parse_args(args)
if len(args) != 2:
op.error('expecting 2 argument')
configFile = _open_config(args[1])
query = dict(subresource='/config')
page = put_page(configFile, SECTION, json.dumps(dict(instance=args[0])),
**query)
if func:
func(page)
return page
def main_write_jobs_to_psuade(args=None):
""" Gets all jobs from the session and writes them to a PSUADE file """
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID PSUADE_IN PSUADE_OUT CONFIG_FILE",
description=main_write_jobs_to_psuade.__doc__)
add_options(op)
(options, args) = op.parse_args()
assert(len(args) >= 4)
sessionid = args[0]
psuadeInFilename = args[1]
psuadeOutFilename = args[2]
configFile = _open_config(args[3])
if(not configFile.has_section(SECTION) or
not configFile.has_option(SECTION, "url") or
configFile.get(SECTION, "url") == ""):
# Local Psuade based run
shutil.copyfile("psuadeData", args[2])
else:
# Gateway run
write_jobs_to_psuade(configFile, sessionid,
psuadeInFilename, psuadeOutFilename, options)
print("Jobs written to", args[2])
def main_list(args=None, func=_print_simulation_list):
"""Retrieves list of all simulations, by default prints in human readable format.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main_list.__doc__)
add_options(op)
add_json_option(op)
(options, args) = op.parse_args(args)
if len(args) != 1:
op.error('expecting 1 argument')
if options.json:
func = _print_json
configFile = _open_config(args[0])
query = {}
if options.verbose:
query['verbose'] = options.verbose
options.page = 1
content = get_page(configFile, SECTION, **query)
data = load_pages_json([content])
if func:
func(data, options.verbose)
return data
def main_delete(args=None, func=_print_page):
"""Delete simulation
"""
op = optparse.OptionParser(
usage="USAGE: %prog SIMULATION_NAME CONFIG_FILE",
description=main_delete.__doc__)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
simulationName = args[0]
try:
configFile = _open_config(*args[1:])
except Exception as ex:
op.error(ex)
log = _log.getLogger(__name__)
try:
page = delete_page(configFile,
SECTION,
subresource='%s' % simulationName)
except HTTPError as ex:
log.error(ex)
raise
log.debug("PAGE: %s" % page)
return page
def main_update_config_floor(args=None, func=_print_page):
"""Sets a floor for the number of Consumer processes that should remain running
for a interval determined by the server. Currently this is only supported in
the AWS EC2 deployment. By default prints entire resultant configuration in JSON.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] INT CONFIG_FILE",
description=main_update_config_floor.__doc__)
# add_options(op)
op.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
help="verbose output")
(options, args) = op.parse_args(args)
if len(args) != 2:
op.error('expecting 2 argument')
configFile = _open_config(args[1])
query = dict(subresource='/config')
page = put_page(configFile, SECTION, json.dumps(dict(floor=int(args[0]))),
**query)
if func:
func(page)
return page
def main_get_config(args=None, func=_print_page):
"""Return configuration settings for top-level Consumer resource, by default
print as JSON. These settings are utilized by an orchestrator process
(deployment specific). The AWS EC2 orchestator handles auto-scaling of instances.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main_get_config.__doc__)
# add_options(op)
op.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
help="verbose output")
(options, args) = op.parse_args(args)
if len(args) != 1:
op.error('expecting 1 argument')
configFile = _open_config(args[0])
query = dict(subresource='/config')
page = get_page(configFile, SECTION, **query)
if func:
func(page)
return page
def main_delete(args=None, func=_print_page):
"""Delete session and all jobs it contains, prints number of jobs deleted as result of session delete.
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_delete.__doc__)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception as ex:
op.error(ex)
log = _log.getLogger(__name__)
try:
page = delete_page(configFile, SECTION, subresource='%s' % sessionid)
except HTTPError as ex:
log.error(ex)
raise
log.debug("PAGE: %s" % page)
return int(page)
def main_session_stats(args=None):
"""session resource utility, prints basic session statistics
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_session_stats.__doc__)
add_options(op)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception as ex:
op.error(ex)
pages = get_paging(configFile, SECTION, options, subresource=sessionid)
data = load_pages_json(pages)
basic_session_stats(sys.stdout, data, verbose=options.verbose)
# if options.verbose:
# for i in success_l:
# print i['Output']
return data
def main_get_session_result_page(args=None, func=_print_page):
"""Terminate jobs in session in state setup, running. Print number of jobs terminated.
"""
op = optparse.OptionParser(
usage="USAGE: %prog SESSIONID PAGE_NUMBER CONFIG_FILE",
description=main_kill_jobs.__doc__)
(options, args) = op.parse_args(args)
if len(args) < 2:
op.error('expecting >= 2 arguments')
sessionid = args[0]
page_number = args[1]
try:
configFile = _open_config(*args[2:])
except Exception as ex:
op.error(ex)
page = get_session_page_results(configFile, sessionid, int(page_number))
if page:
page = json.loads(page)
#data = int(page)
if func:
func(page)
return page
def setUp(self):
""" Open up configuration and setup logging before doing anything.
"""
self.cp = cp = _open_config(self.config_name)
_setup_logging(cp)
self.log = logging.getLogger(
'%s.%s.%s' %
(self._module_name, self.__class__.__name__, self._testMethodName))
def main_create(args=None):
"""Create an empty Simulation Resource
"""
op = optparse.OptionParser(
usage=
"USAGE: %prog [options] SIMULATION_NAME|GUID APPLICATION_NAME CONFIG_FILE",
description=main_create.__doc__)
op.add_option(
"-s",
"--simulation",
action="store",
dest="simulation_name",
default=None,
help=
"override versioning system and specify the GUID identifier for a simulation "
)
(options, args) = op.parse_args(args)
if len(args) != 3:
op.error('expecting 3 arguments')
cp = _open_config(args[2])
application = args[1]
subresource = args[0]
simulation_name = None
if options.simulation_name:
uuid.UUID(subresource)
simulation_name = options.simulation_name
else:
try:
uuid.UUID(subresource)
except ValueError:
pass
else:
op.error('must use -s option when specifying a guid, see help')
simulation_name = subresource
kw = dict(subresource=subresource)
data = json.dumps(
dict(Application=application, StagedInputs=[], Name=simulation_name))
try:
data = put_page(cp,
SECTION,
data,
content_type='application/json',
**kw)
except HTTPError as ex:
_log.error(ex)
if hasattr(ex, 'readlines'):
_log.error("".join(ex.readlines()))
elif hasattr(ex, 'read'):
_log.error("".join(ex.read()))
raise
return json.loads(data)
def main(args=None, func=_print_as_json):
"""Queries for job resources based on select criteria, by default prints JSON array of jobs.
"""
global states
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main.__doc__)
op.add_option("-j", "--jobid",
action="store", dest="subresource", default=None,
help="JOB ID")
op.add_option("-n", "--sim",
action="store", dest="simulation", default=None,
help="Simulation Name")
op.add_option("-x", "--state",
action="store", dest="state", default=None,
help="Job Status to query: %s" %list(states))
op.add_option("-c", "--consumer",
action="store", dest="consumer", default=None,
help="Consumer GUID to query")
op.add_option("-b", "--basic",
action="store_true", dest="basic",
help="Print Basic Information About Job(s)")
add_options(op)
add_session_option(op)
(options, args) = op.parse_args(args)
configFile = _open_config(args[0])
query = {}
if options.session: query['session'] = options.session
if options.simulation: query['simulation'] = options.simulation
if options.subresource: query['subresource'] = options.subresource
if options.state: query['state'] = options.state
if options.consumer: query['consumer'] = options.consumer
if options.verbose: query['verbose'] = options.verbose
if options.subresource:
page = get_page(configFile, SECTION, **query)
job = json.loads(page)
if options.basic:
write_basic_job_info(sys.stdout, job, verbose=options.verbose)
elif func is not None and callable(func):
func(job, sys.stdout)
return job
try:
pages = get_paging(configFile, SECTION, options, **query)
except HTTPError, ex:
print ex
print ex.readlines()
return
def main_create_session(args=None, func=_print_page):
"""Creates new session resource and prints GUID of created session.
"""
op = optparse.OptionParser(usage="USAGE: %prog CONFIG_FILE",
description=main_create_session.__doc__)
(options, args) = op.parse_args(args)
try:
configFile = _open_config(*args)
except Exception, ex:
op.error(ex.Message)
def main_update(args=None):
"""Update simulation by essentially doing a PUT with the specified file to the resource or optionally sub-resource.
"""
op = optparse.OptionParser(
usage=
"USAGE: %prog [options] SIMULATION_NAME|GUID FILE_NAME CONFIG_FILE",
description=main_update.__doc__)
op.add_option(
"-r",
"--resource",
action="store",
dest="resource",
help="select the staged input file name (matches Application)")
(options, args) = op.parse_args(args)
if len(args) != 3:
op.error('expecting 3 arguments')
log = _log.getLogger('%s.main_update' % __name__)
log.debug(args)
file_name = args[1]
if not os.path.isfile(file_name):
op.error('expecting a file for argument 2')
if options.resource is None:
op.error('require resource option to be specified')
configFile = _open_config(args[2])
simulation = args[0]
kw = {}
out = sys.stdout
kw['subresource'] = '%s/input/%s' % (simulation, options.resource)
with open(file_name, 'rb') as fd:
contents = fd.read()
try:
data = put_page(configFile, SECTION, contents, **kw)
except HTTPError as ex:
_log.error("HTTP Code %d : %s", ex.code, ex.msg)
if hasattr(ex, 'readlines'):
_log.debug("".join(ex.readlines()))
else:
_log.debug("".join(ex.read()))
raise
except urllib.error.URLError as ex:
_log.error("URLError : %s", ex.reason)
raise
return data
def main_list(args=None, func=_print_numbered_lines):
"""Prints human readable listing of all session GUIDs.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main_list.__doc__)
add_options(op)
add_json_option(op)
(options, args) = op.parse_args(args)
try:
configFile = _open_config(*args)
except Exception, ex:
op.error(ex.Message)
def main_get_consumer_by_guid(args=None, func=_print_page):
"""Retrieves consumer by GUID
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONSUMER_GUID CONFIG_FILE",
description=main.__doc__)
(options, args) = op.parse_args(args)
if len(args) != 2:
op.error('expecting 2 arguments')
configFile = _open_config(args[1])
query = dict(subresource='/%s' %args[0])
page = get_page(configFile, SECTION, **query)
data = json.loads(page)
if func: func(data)
return data
def main_start_jobs(args=None, func=_print_page):
"""Move all jobs in states pause and create to submit. Prints the number of jobs moved to state submit.
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_start_jobs.__doc__)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception, ex:
op.error(ex.Message)
def main_csv_launch(args=None):
"""Takes a CSV file and submits all the jobs contained therein as a new session. Returns immediatly."""
op = optparse.OptionParser(usage="USAGE: %prog CSV_IN CONFIG_FILE",
description=main_csv_launch.__doc__)
(options, args) = op.parse_args()
assert (len(args) == 2)
csvInFilename = args[0]
configFilename = args[1]
configFile = _open_config(configFilename)
print(csv_launch(configFile, csvInFilename))
def main_kill_jobs(args=None, func=_print_page):
"""Terminate jobs in session in state setup, running. Print number of jobs terminated.
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_kill_jobs.__doc__)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception, ex:
op.error(ex.Message)
def main_delete(args=None, func=_print_page):
"""Delete session and all jobs it contains, prints number of jobs deleted as result of session delete.
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_delete.__doc__)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception, ex:
op.error(ex.Message)
def getNumUnfinishedRunSamples():
if not LocalExecutionModule.runStarted:
return -1
elif LocalExecutionModule.runComplete:
return 0
if not os.path.exists('psuadePath'):
return -1
if LocalExecutionModule.runType == Model.LOCAL:
print('getting config')
config = _open_config(LocalExecutionModule.configFile)
else:
config = None
numUnfinished = turbine_psuade_session_script.local_unfinished(
config, 'psuadeData')
if numUnfinished == 0:
LocalExecutionModule.runComplete = True
LocalExecutionModule.stdoutF.close()
LocalExecutionModule.stderrF.close()
#Clean up files
psuadeAppsFiles = os.getcwd() + os.path.sep + 'psuadeApps_ct.*'
for f in glob.glob(psuadeAppsFiles):
os.remove(f)
## # Check if process is stopped with errors
## elif LocalExecutionModule.process.state() != QtCore.QProcess.Running:
## ##elif LocalExecutionModule.process.poll() is not None: #Process done
## LocalExecutionModule.stdoutF.close()
## LocalExecutionModule.stderrF.close()
#Clean up files
psuadeAppsFiles = os.getcwd() + os.path.sep + 'psuadeApps_ct.*'
for f in glob.glob(psuadeAppsFiles):
os.remove(f)
## #Open file and assert error from psuade
## errorOutput = LocalExecutionModule.process.readAllStandardError()
## print errorOutput
## standardOutput = LocalExecutionModule.process.readAllStandardOutput()
## print standardOutput
#### outF = open(os.getcwd() + '\\stdout')
#### lines = outF.readlines()
## raise RuntimeError(lines)
return numUnfinished
def main_log(args=None, func=_print_page):
"""Retrieves logging messages from compute resource running the specified
Consumer. Log messages are printed to screen in order. This functionality
is not available in all deployments.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main_log.__doc__)
(options, args) = op.parse_args(args)
if len(args) != 2:
op.error('expecting 2 arguments')
configFile = _open_config(args[1])
query = dict(subresource='/%s/log' %args[0])
page = get_page(configFile, SECTION, **query)
if func: func(page)
return page
def main_session_stats(args=None):
"""session resource utility, prints basic session statistics
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_session_stats.__doc__)
add_options(op)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception, ex:
op.error(ex.Message)
def main_delete(args=None, func=_print_page):
"""Delete simulation
"""
op = optparse.OptionParser(
usage="USAGE: %prog SIMULATION_NAME CONFIG_FILE",
description=main_delete.__doc__)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
simulationName = args[0]
try:
configFile = _open_config(*args[1:])
except Exception, ex:
op.error(ex.Message)
def main(args=None, func=_print_numbered_lines):
"""List all Consumer resources, by default print in human readable format.
"""
op = optparse.OptionParser(usage="USAGE: %prog [options] CONFIG_FILE",
description=main.__doc__)
# add_options(op)
op.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
help="verbose output")
"""
op.add_option("-p", "--page", type="int",
action="store", dest="page", default=0,
help="page number")
op.add_option("-r", "--rpp", type="int",
action="store", dest="rpp", default=1000,
help="results per page")
"""
op.add_option("-s",
"--status",
action="store",
dest="status",
help="query on status ['up'|'down'|'error']")
add_json_option(op)
(options, args) = op.parse_args(args)
if len(args) != 1:
op.error('expecting 1 argument')
configFile = _open_config(args[0])
query = {}
if options.status:
query['status'] = options.status
# NOTE RESOURCE NOT SUPPORTING PAGING
#pages = get_paging(configFile, SECTION, options, **query)
#data = load_pages_json(pages)
page = get_page(configFile, SECTION, **query)
data = json.loads(page)
if options.json:
func = _print_as_json
if func:
func(data)
return data
def main_create_jobs(args=None, func=_print_page):
"""Appends job descriptions to a session, prints the number of jobs added.
"""
op = optparse.OptionParser(
usage="USAGE: %prog SESSIONID JOBS_FILE CONFIG_FILE",
description=main_create_jobs.__doc__)
(options, args) = op.parse_args(args)
if len(args) < 2:
op.error('expecting >= 2 arguments')
sessionid = args[0]
jobs_file = args[1]
try:
configFile = _open_config(*args[2:])
except Exception as ex:
op.error(ex)
log = _log.getLogger(__name__)
log.debug("main_create_jobs")
try:
with open(jobs_file) as fd:
input_data = json.load(fd)
except FileNotFoundError:
input_data = json.loads(jobs_file)
assert (type(input_data) is list)
for o in input_data:
thisset = set(o.keys())
assert thisset.issuperset([
"Input", "Simulation"
]), "Each entry requires fields 'Input' and 'Simulation'"
try:
page = create_jobs(configFile, sessionid, input_data)
except HTTPError as ex:
log.error(ex)
log.error(ex.fp.read())
raise
log.debug("PAGE: %s" % page)
data = json.loads(page)
if (func):
func(data)
return data
def main_jobs_status(args=None, func=_print_as_json):
"""session resource utility, lists all session resources
"""
op = optparse.OptionParser(
usage="USAGE: %prog [options] SESSIONID CONFIG_FILE",
description=main_jobs_status.__doc__)
add_options(op)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 argument')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception, ex:
op.error(ex.Message)
def main_stop_jobs(args=None):
"""Move all jobs in state submit to pause. Prints the number of jobs moved to state pause.
"""
op = optparse.OptionParser(usage="USAGE: %prog SESSIONID CONFIG_FILE",
description=main_stop_jobs.__doc__)
(options, args) = op.parse_args(args)
if len(args) < 1:
op.error('expecting >= 1 arguments')
sessionid = args[0]
try:
configFile = _open_config(*args[1:])
except Exception as ex:
op.error(ex)
print(stop_jobs(configFile, sessionid))
