def makePlotPath(value, simId):
plotDir = getParam('MCS.PlotDir')
subDir = os.path.join(plotDir, "s%d" % simId)
mkdirs(subDir)
plotType = getParam('MCS.PlotType')
path = os.path.join(subDir, "%s.%s" % (value, plotType))
#print "Plot path: ", path
return path
def addArgs(self, parser):
from pygcam.config import getParam #, getParamAsInt
# defaultQueue = getParam('IPP.Queue')
defaultProfile = getParam('IPP.Profile')
defaultClusterId = getParam('IPP.ClusterId')
defaultWorkDir = getParam('IPP.WorkDir')
defaultNumTrials = 1
parser.add_argument(
'-c',
'--clusterId',
type=str,
default=defaultClusterId,
help='''A string to identify this cluster. Default is the
value of config var IPP.ClusterId, currently
"%s".''' % defaultClusterId)
parser.add_argument(
'-n',
'--numTrials',
type=int,
default=defaultNumTrials,
help='''The number of additional trials to create engines for.
Default is %d''' % defaultNumTrials)
parser.add_argument(
'-o',
'--otherArgs',
type=str,
default='',
help='Command line arguments to append to the ipengine command.')
parser.add_argument(
'-p',
'--profile',
type=str,
default=defaultProfile,
help='''The name of the ipython profile to use. Default is
the value of config var IPP.Profile, currently
"%s".''' % defaultProfile)
# parser.add_argument('-q', '--queue', type=str, default=defaultQueue,
# help='''The queue or partition on which to create the controller
# and engines. Overrides config var IPP.Queue, currently
# "%s".''' % defaultQueue)
parser.add_argument(
'-w',
'--workDir',
type=str,
default=defaultWorkDir,
help='''Where to run the ipengine command. Overrides the
value of config var IPP.WorkDir, currently '%s'.'''
% defaultWorkDir)
return parser # for auto-doc generation
def driver(args, tool):
'''
Generate a simulation. Do generic setup, then call genSimulation().
'''
from pygcam.utils import removeTreeSafely
from ..Database import getDatabase
from ..error import PygcamMcsUserError
from ..util import saveDict
paramFile = args.paramFile or getParam('MCS.ParametersFile')
if args.exportVars:
_exportVars(paramFile, args.exportVars)
return
simId = args.simId
desc = args.desc
trials = args.trials
if trials < 0:
raise PygcamMcsUserError(
"Trials argument is required: must be an integer >= 0")
projectName = args.projectName
runRoot = args.runRoot
if runRoot:
# TBD: write this to config file under [project] section
setParam('MCS.Root', runRoot, section=projectName)
_logger.info(
'Please add "MCS.Root = %s" to your .pygcam.cfg file in the [%s] section.',
runRoot, projectName)
runDir = getParam('MCS.RunDir', section=projectName)
if args.delete:
removeTreeSafely(runDir, ignore_errors=False)
runWorkspace = getParam('MCS.RunWorkspace')
if not os.path.exists(runWorkspace):
_newsim(runWorkspace, trials)
# Called with trials == 0 when setting up a local run directory on /scratch
if trials:
# The simId can be provided on command line, in which case we need
# to delete existing parameter entries for this app and simId.
db = getDatabase()
simId = db.createSim(trials, desc, simId=simId)
genSimulation(simId, trials, paramFile, args=args)
if trials:
# Save a copy of the arguments used to create this simulation
simDir = getSimDir(simId)
argSaveFile = '%s/gcamGenSimArgs.txt' % simDir
saveDict(vars(args), argSaveFile)
def __init__(self,
runId=None,
projectName=None,
simId=None,
trialNum=None,
scenario=None,
baseline=None,
groupName=None,
status=None,
store=True):
self.runId = runId
self.simId = simId
self.trialNum = trialNum
self.scenario = scenario
self.baseline = baseline
self.status = status
self.projectName = projectName = projectName or getParam(
'GCAM.DefaultProject')
project = Project.readProjectFile(projectName)
self.groupName = groupName or project.scenarioSetup.defaultGroup
self.useGroupDir = project.scenarioGroup.useGroupDir
if store and runId:
self.saveRunInfo()
def _runPygcamSteps(steps, context, runWorkspace=None, raiseError=True):
"""
run "gt +P {project} --mcs=trial run -s {step[,step,...]} -S {scenarioName} ..."
For Monte Carlo trials.
"""
import pygcam.tool
runWorkspace = runWorkspace or getParam('MCS.RunWorkspace')
trialDir = context.getTrialDir()
groupArg = ['-g', context.groupName] if context.groupName else []
# N.B. MCS.RunWorkspace is the RefWorkspace for trial sandboxes
toolArgs = [
'+P', context.projectName, '--mcs=trial',
'--set=GCAM.SandboxRefWorkspace=' + runWorkspace, 'run', '-s', steps,
'-S', context.scenario, '--sandboxDir=' + trialDir
] + groupArg
command = 'gt ' + ' '.join(toolArgs)
_logger.debug('Running: %s', command)
status = pygcam.tool.main(argv=toolArgs, raiseError=True)
msg = '"%s" exited with status %d' % (command, status)
if status != 0 and raiseError:
raise GcamToolError(msg)
_logger.info("_runSteps: " + msg)
return status
def driver(args, tool):
import os
'''
Set up the SQL database. Must call startDb() before calling this.
'''
import shutil
from pygcam.config import getParam
from ..Database import getDatabase
from ..error import PygcamMcsSystemError
if args.deleteSims:
# Remove the whole sims dir and remake it
runSimsDir = getParam('MCS.RunSimsDir')
if os.path.exists(runSimsDir):
try:
shutil.rmtree(runSimsDir)
os.mkdir(runSimsDir)
except Exception as e:
raise PygcamMcsSystemError(
'Failed to recreate sim dir %s: %s' % (runSimsDir, e))
db = getDatabase()
# reinitialize the db
db.initDb(args=args)
if not (args.empty):
from ..XMLResultFile import XMLResultFile
XMLResultFile.addOutputs()
def _newsim(runWorkspace, trials):
'''
Setup the app and run directories for a given user app.
'''
from pygcam.scenarioSetup import copyWorkspace
from ..Database import getDatabase
from ..error import PygcamMcsUserError
from ..XMLResultFile import XMLResultFile
if not runWorkspace:
raise PygcamMcsUserError(
"MCS.RunWorkspace was not set in the configuration file")
srcDir = getParam('GCAM.RefWorkspace')
dstDir = runWorkspace
copyWorkspace(dstDir, refWorkspace=srcDir, forceCreate=True, mcsMode=True)
if trials:
db = getDatabase() # ensures database initialization
XMLResultFile.addOutputs()
# Load SQL script to create convenient views
text = getResource('mcs/etc/views.sql')
db.executeScript(text=text)
def projectChooser(self):
layout = dcc.Dropdown(id='project-chooser',
options=[{
'label': name,
'value': name
} for name in self.projects],
value=getParam('GCAM.DefaultProject'))
return layout
def getRunQueryDir():
"""
Returns the path to sim's copy of the scenarios.xml file.
"""
workspace = getParam('MCS.RunWorkspace')
path = os.path.join(workspace, QueryDirName)
return path
def readProjectFile(cls, projectName, groupName=None, projectFile=None):
# return cached project if already read, otherwise read project.xml
if not cls.instance or cls.instance.projectName != projectName:
projectFile = projectFile or getParam('GCAM.ProjectXmlFile',
section=projectName)
cls.instance = Project(projectFile, projectName, groupName)
return cls.instance
def projectsWithDatabases():
withDatabases = []
for project in getSections():
dbPath = getParam('MCS.DbPath', section=project, raiseError=False)
if dbPath and os.path.exists(dbPath):
withDatabases.append(project)
return withDatabases
def _readParameterInfo(context, paramPath):
from pygcam.xmlSetup import ScenarioSetup
scenarioFile = getParam('GCAM.ScenarioSetupFile')
scenarioSetup = ScenarioSetup.parse(scenarioFile)
scenarioNames = scenarioSetup.scenariosInGroup(context.groupName)
paramFile = XMLParameterFile(paramPath)
paramFile.loadInputFiles(context, scenarioNames, writeConfigFiles=False)
paramFile.runQueries()
return paramFile
def driver(args, tool):
"""
Start ipyparallel engines.
"""
from pygcam.config import getParam
from ..master import startEngines, templatePath
scheduler = getParam('IPP.Scheduler')
batchTemplate = templatePath(scheduler, args.profile, args.clusterId,
'engine')
startEngines(args.numTrials, batchTemplate)
def _setup_werkzeug_log(level, host, port):
import logging
consoleFormat = getParam('GCAM.LogConsoleFormat')
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(consoleFormat))
log = logging.getLogger('werkzeug')
log.addHandler(handler)
log.setLevel(level)
log.propagate = False
# Setting Flask.LogLevel > INFO quashes this useful message, so we restore it
if log.level > logging.INFO:
print('* Running on http://{}:{}/ (Press CTRL+C to quit)'.format(
host, port))
def createOutputDir(outputDir):
from ..utils import removeFileOrTree
from ..temp_file import getTempDir
removeFileOrTree(outputDir, raiseError=False)
tempOutputDir = getParam('MCS.TempOutputDir')
if tempOutputDir:
# We create this on /scratch which is purged automatically.
newDir = getTempDir(suffix='', tmpDir=tempOutputDir, delete=False)
mkdirs(newDir)
_logger.debug("Creating '%s' link to %s" % (outputDir, newDir))
symlink(newDir, outputDir)
else:
mkdirs(outputDir)
def getSimDir(simId, create=False):
'''
Return and optionally create the path to the top-level simulation
directory for the given simulation number, based on the SimsDir
parameter specified in the config file.
'''
simsDir = getParam('MCS.RunSimsDir')
if not simsDir:
raise PygcamMcsUserError(
"Missing required config parameter 'RunSimsDir'")
simDir = os.path.join(simsDir,
's%03d' % simId) # name is of format ".../s001/"
if create:
mkdirs(simDir)
return simDir
def run(self, args, tool):
# If not passed on command-line, read from config file
yearsStr = args.years or getParam('GCAM.Years')
years = [int(s) for s in yearsStr.split('-')]
if len(years) != 2:
raise PygcamException('''Years must be specified as XXXX-YYYY, where XXXX and YYYY
are the first and last years to consider, respectively''')
firstYear, lastYear = years
guess = shockedFuel(args.policy)
_logger.info("Inferred shocked fuel '%s' from policy '%s'", guess, args.policy)
shocked = [guess]
GcamResultProcessor(args.baseline, args.policy, args.diffsDir, shocked,
args.fuelShockFile, firstYear=firstYear, lastYear=lastYear)
def main():
import pandas as pd
ref_ws = getParam('GCAM.RefWorkspace')
data_sys = pathjoin(ref_ws, 'input', 'gcam-data-system')
aglu_xml = pathjoin(data_sys, 'xml', 'aglu-xml')
protected_land_2_xml = pathjoin(aglu_xml, 'protected_land_input_2.xml')
protected_land_3_xml = pathjoin(aglu_xml, 'protected_land_input_3.xml')
parser = ET.XMLParser(remove_blank_text=True, remove_comments=True)
prot2 = ET.parse(protected_land_2_xml, parser)
prot3 = ET.parse(protected_land_3_xml, parser)
regionMap = pd.read_csv(pathjoin(data_sys, '_common', 'mappings', 'GCAM_region_names_32reg.csv'), skiprows=3)
regionMap.set_index('GCAM_region_ID', inplace=True)
region = regionMap.region
# Read the CSV defining the region-wide land protection levels for OTAQ project and
# apply this anew, verifying that the only differences are way into the decimal places
otaq_prot_file = pathjoin(ref_ws, 'input', 'otaq-modifications', 'aglu', 'OTAQ_land_protection.csv')
otaq_prot_df = pd.read_csv(otaq_prot_file)
otaq_prot_df['region'] = otaq_prot_df['GCAM_region_ID'].map(region)
otaq_prot_df.drop('GCAM_region_ID', axis=1, inplace=True)
otaq_prot_df.set_index('region', inplace=True)
fracs = otaq_prot_df.protect_land_fract
# protected_land_2 has only UnmanagedPasture
prot2_dict = {reg: [('UnmanagedPasture', fracs[reg])] for reg in fracs.index}
_protect_land(prot2, prot2_dict)
outfile = "/Users/rjp/Downloads/land_prot2_modified.xml"
print("Writing", outfile)
prot2.write(outfile, xml_declaration=True, pretty_print=True)
# protected_land_3 has all the other unmanaged types
landtypes = ['UnmanagedForest', 'Grassland', 'Shrubland']
prot3_dict = {reg: [(landtype, fracs[reg]) for landtype in landtypes] for reg in fracs.index}
_protect_land(prot3, prot3_dict)
outfile = "/Users/rjp/Downloads/land_prot3_modified.xml"
print("Writing", outfile)
prot3.write(outfile, xml_declaration=True, pretty_print=True)
def main():
args = parseArgs()
collection_file = Path(args.collection_file)
if collection_file.exists():
collection_file.rename(collection_file.name + '~')
wrapper_file = args.wrapper_file
sandbox_dir = Path(args.sandbox_dir or getParam('GCAM.SandboxDir'))
with open(collection_file, 'w') as output:
paths = sandbox_dir.glob(f'*/exe/{wrapper_file}')
for p in paths:
scenario = p.parent.parent.name
with open(p, 'r') as input:
lines = input.readlines()
for line in lines:
output.write(f'{scenario},{line}')
def __init__(self,
filename,
load=True,
schemaPath=None,
removeComments=True,
conditionalXML=False,
varDict=None):
self.filename = filename
self.tree = None
self.conditionalXML = conditionalXML
self.varDict = varDict or getConfigDict(
section=getParam('GCAM.DefaultProject'))
self.removeComments = removeComments
self.schemaPath = schemaPath
self.schemaStream = None
if filename and load:
self.read()
def readMetaData(self, project=None):
if project:
setParam('GCAM.DefaultProject', project, DEFAULT_SECTION)
setSection(project)
else:
project = getParam('GCAM.DefaultProject')
self.project = project
self.inputsDF = None
self.resultDict = None
self.selectedResults = None
self.corrDF = None
self.simId = None
if self.db:
self.db.close()
db = self.db = getDatabase()
self.inputs = db.getInputs()
def activeYears(asInt=False):
'''
Convert a string identifying active years into a list of ints or strs. Values must
be comma-separated integers or expressions of the form xxxx-yyyy or xxxx-yyyy:z. The
two expressions indicate ranges of years, with a default timestep of 5 years. If
given, the final value after the colon indicates an alternative timestep.
'''
global _activeYearInts
global _activeYearStrs
# return cached values or compute and cache result
if not _activeYearStrs:
import re
from functools import reduce
def reducer(lst, item):
m = re.match(r"^(\d{4})-(\d{4})(?::(\d+))?$", item)
if m:
start = int(m.group(1))
end = int(m.group(2))
step = int(m.group(
3)) if m.lastindex == 3 else 5 # default 5 yr timestep
rng = list(range(start, end + 1, step))
lst.extend(rng)
elif item.isdigit():
lst.append(int(item))
else:
raise PygcamMcsUserError(
'Element in list of active years is not an integer: %s' %
item)
return lst
yearStr = getParam('MCS.Years')
items = yearStr.split(',')
years = reduce(reducer, items, [])
_activeYearInts = [int(y) for y in years]
_activeYearStrs = [str(y) for y in years]
return _activeYearInts if asInt else _activeYearStrs
def parseArgs():
parser = argparse.ArgumentParser(
description=
'''Collect files written by custom GCAM wrapper into a single file
with each line containing the name of scenario generating the file.'''
)
wrapper_file = './failed_periods.txt'
parser.add_argument(
'-f',
'--wrapper_file',
default=wrapper_file,
help=
f'''The name of the file created by the custom GCAM wrapper function.
Default is '{wrapper_file}'.''')
collection_file = './collected_wrapper_output.txt'
parser.add_argument(
'-o',
'--collection_file',
default=collection_file,
help=f'''The directory into which all normalized XML files are written.
Default is '{collection_file}'.''')
sandbox_dir = getParam('GCAM.SandboxDir')
parser.add_argument(
'-s',
'--sandbox_dir',
default=None,
help=
f'''The directory containing folders for the scenarios from which the wrapper output
files should be collected. Default is taken from pygcam config variable "GCAM.SandboxDir",
currently "{sandbox_dir}".''')
args = parser.parse_args()
return args
def driver(args, tool):
"""
Analyze MCS results
"""
import os
from ..analysis import analyzeSimulation
from ..error import PygcamMcsUserError
if args.timeseries:
import pandas as pd
from pygcam.config import getParam
from ..Database import getDatabase
from ..timeseriesPlot import plotTimeSeries, plotForcingSubplots
from ..util import stripYearPrefix
simId = args.simId
expList = args.expName.split(',')
resultName = args.resultName
xlabel = 'Year' # args.xlabel or 'Year'
ymin = args.ymin
ymax = args.ymax
# special purpose plot
forcingPlot = args.forcingPlot
if not (expList[0] and resultName):
raise PygcamMcsUserError(
"expName and resultName must be specified")
db = getDatabase()
trialCount = db.getTrialCount(simId)
plotDir = getParam('MCS.PlotDir')
plotType = getParam('MCS.PlotType')
allResults = db.getTimeSeries(simId, resultName,
expList) # , regionName)
if not allResults:
raise PygcamMcsUserError('No timeseries results for simId=%d, expList=%s, resultName=%s' \
% (simId, expList, resultName))
def computeFilename(expName):
basename = "%s-s%d-%s.%s" % (resultName, simId, expName, plotType)
filename = os.path.join(plotDir, 's%d' % simId, basename)
return filename
# massage the data into the format required by plotTimeSeries
def createRecord(pair):
obj, expName = pair
d = obj.__dict__
d['expName'] = expName
return d
records = [createRecord(pair) for pair in allResults]
resultDF = pd.DataFrame.from_records(records, index='seriesId')
units = resultDF.units.iloc[0]
resultDF.drop(['units', '_sa_instance_state', 'outputId'],
axis=1,
inplace=True)
# convert column names like 'y2020' to '2020'
cols = [stripYearPrefix(c) for c in resultDF.columns]
resultDF.columns = cols
if forcingPlot:
filename = computeFilename('combo')
plotForcingSubplots(resultDF,
filename=filename,
ci=95,
show_figure=False)
return
for expName in expList:
# create a copy so we can drop expName column for melt
df = resultDF.query("expName == '%s'" % expName).copy()
_logger.debug("Found %d result records for exp %s, result %s" %
(len(df), expName, resultName))
df.drop(['expName'], axis=1, inplace=True)
df = pd.melt(df, id_vars=['runId'], var_name='year')
title = '%s for %s' % (resultName, expName)
filename = computeFilename(expName)
_logger.debug('Saving timeseries plot to %s' % filename)
reg = "" # '-' + regionName if regionName else ""
extra = "name=%s trials=%d/%d simId=%d scenario=%s%s" % \
(resultName, resultDF.shape[0], trialCount, simId, expName, reg)
# TBD: generalize this with a lookup table or file
if units == 'W/m^2':
units = 'W m$^{-2}$'
plotTimeSeries(
df,
'year',
'runId',
title=title,
xlabel=xlabel,
ylabel=units,
ci=[95], # [50, 95],
text_label=None,
legend_name=None,
legend_labels=None,
ymin=ymin,
ymax=ymax,
filename=filename,
show_figure=False,
extra=extra)
# if doing timeseries plot, none of the other options are relevant
return
if not (args.exportInputs or args.resultFile or args.plot
or args.importance or args.groups or args.plotInputs or args.stats
or args.convergence or args.exportEMA, args.exportAll):
msg = 'Must specify at least one of: --export, --resultFile, --plot, --importance, --groups, --distros, --stats, --convergence, --exportEMA, --exportAll'
raise PygcamMcsUserError(msg)
analyzeSimulation(args)
def main(args):
app = dash.Dash(name='mcs-explorer', csrf_protect=False)
level = getParam('Flask.LogLevel')
flaskLog = app.server.logger
flaskLog.setLevel(level)
_setup_werkzeug_log(level, args.host, args.port)
# app.config.supress_callback_exceptions = True
app.css.append_css({
"external_url":
"https://fonts.googleapis.com/css?family=Dosis:regular,bold|Lato"
})
app.css.append_css(
{"external_url": "https://codepen.io/plevin/pen/MvpeNV.css"})
data = McsData(app)
data.readMetaData()
app.layout = data.layout()
#
# Callbacks
#
generateDropdownDefaults(app, ['scenario-chooser', 'output-chooser'])
@app.callback(Output('output-chooser', 'options'), [
Input('project-chooser', 'value'),
Input('sim-chooser', 'value'),
Input('scenario-chooser', 'value')
])
def updateOutputChooser(project, simId, scenario):
_logger.debug('updateOutputChooser(%s, %s, %s)' %
(project, simId, scenario))
layout = data.outputChooser(simId=simId,
scenario=scenario,
optionsOnly=True)
return layout
@app.callback(Output('multi-output-chooser', 'options'), [
Input('project-chooser', 'value'),
Input('sim-chooser', 'value'),
Input('scenario-chooser', 'value')
])
def updateMultiOutputChooser(project, simId, scenario):
_logger.debug('updateMultiOutputChooser(%s, %s, %s)' %
(project, simId, scenario))
layout = data.multiOutputChooser(simId=simId,
scenario=scenario,
optionsOnly=True)
return layout
@app.callback(
Output('scenario-chooser', 'options'),
[Input('project-chooser', 'value'),
Input('sim-chooser', 'value')])
def updateScenarioChooser(project, simId):
_logger.debug('updateScenarioChooser(%s, %s)' % (project, simId))
layout = data.scenarioChooser(simId, multi=False, optionsOnly=True)
return layout
@app.callback(Output('sim-chooser', 'options'),
[Input('project-chooser', 'value')])
def updateSimChooser(project):
_logger.debug('updateSimChooser(%s)' % project)
data.readMetaData(project)
layout = data.simChooser(optionsOnly=True)
return layout
# TBD: finish writing these methods to store data in hidden div for
# TBD: reset button and selection to simplify callback for drawing dist
#
# The next few callbacks use a hidden <div> to store
# communicate the variables used in the parallel coords
# figure with those in the output chooser.
#
@app.callback(Output('paraCoords-vars', 'children'),
[Input('paraCoords', 'figure')])
def updateParaCoordsVars(figure):
varNames = data.paraCoordsVars
_logger.debug('updateParaCoordsVars vars=%s' % varNames)
varNamesJSON = json.dumps(varNames)
return varNamesJSON
@app.callback(Output('input-chooser', 'value'),
[Input('paraCoords-vars', 'children')])
def updateInputChooser(varNamesJSON):
varNames = json.loads(varNamesJSON)
_logger.debug('updateInputChooser(%s)' % varNames)
return varNames
@app.callback(
Output('paraCoords', 'figure'),
[
Input('project-chooser', 'value'),
Input('sim-chooser', 'value'),
Input('scenario-chooser', 'value'),
Input('output-chooser', 'value'),
Input('paraCoords-slider', 'value'),
Input('distribution', 'figure'), # force this to plot last
])
def showParallelCoords(project, simId, scenario, resultName, varsToShow,
figure):
return data.parcoordsPlot(simId, scenario, resultName, varsToShow)
@app.callback(Output('percentile-text', 'children'),
[Input('dist-slider', 'value')])
def showDistSliderValues(sliderInfo):
return '%d%%-%d%%' % tuple(sliderInfo)
@app.callback(Output('tornado', 'figure'), [
Input('project-chooser', 'value'),
Input('sim-chooser', 'value'),
Input('scenario-chooser', 'value'),
Input('output-chooser', 'value'),
Input('tornado-slider', 'value'),
Input('tornado-type-chooser', 'value'),
Input('distribution', 'selectedData'),
])
def showTornado(project, simId, scenario, resultName, sliderValue,
tornadoType, selectedData):
_logger.debug('showTornado(%s, %s, %s, %s, %s, %s, %s)' %
(project, simId, scenario, resultName, sliderValue,
tornadoType, selectedData))
if not (project and scenario and resultName) or simId is None:
return ''
figure = data.tornadoPlot(simId, scenario, resultName, tornadoType,
sliderValue, selectedData)
return figure
@app.callback(Output('distribution', 'figure'), [
Input('project-chooser', 'value'),
Input('sim-chooser', 'value'),
Input('scenario-chooser', 'value'),
Input('output-chooser', 'value'),
Input('dist-slider', 'value'),
Input('dist-options', 'values'),
Input('distribution', 'selectedData'),
])
def showDistribution(project, simId, scenario, outputName, sliderInfo,
distOptions, selectedData):
_logger.debug('showDistribution(%s, %s, %s, %s, %s)' %
(project, simId, scenario, outputName, selectedData))
# Clear selectedData if context changes
context = (project, simId, scenario, outputName)
if not data.histogramContext or context != data.histogramContext:
_logger.debug('showDistribution: clearing context')
selectedData = sliderInfo = None
data.histogramContext = context
# if called before values are known, return an empty plot
if not (project and scenario and outputName) or simId is None:
plotData = {'x': []}
title = 'Please select a model output to plot'
annotations = None
else:
plotData, title, annotations = data.distPlot(
simId, scenario, outputName, sliderInfo, distOptions,
selectedData)
# TBD: generalize this
if Oct16:
if outputName == 'percent-change':
xtitle = 'Change from baseline fuel'
xticksuffix = '%'
else:
# latex formatting is broken, but HTML works fine
# xtitle = '$g CO_2 MJ^{-1}$' if Oct16 else ''
xtitle = 'g CO<sub>2</sub> MJ<sup>-1</sup>'
xticksuffix = None
else:
xtitle = data.db.getOutputUnits(outputName)
xticksuffix = ''
layout = updateStyle('Plot',
title=title,
yaxis={
'title': 'Probability density',
'tickvals': []
},
xaxis={
'title': xtitle,
'ticksuffix': xticksuffix,
},
margin=getStyle('PlotMarginWithXTitle'),
dragmode='select',
showLegend=True,
legend=dict(x=0.0,
y=1.0,
bgcolor=getColor('PlotBg')))
if annotations:
layout['annotations'] = annotations
figure = dict(data=plotData, layout=layout)
return figure
# dist -> dist slider
@app.callback(Output('dist-slider', 'value'), [
Input('output-chooser', 'value'),
Input('distribution', 'selectedData'),
Input('dist-slider-reset-button', 'n_clicks')
], [State('sim-chooser', 'value'),
State('scenario-chooser', 'value')])
def showPercentileSlider(resultName, selected, clicks, simId, scenario):
_logger.debug('showPercentileSlider(%s, %s, %s)' %
(resultName, selected, clicks))
if clicks != data.resetSliderButtonClicks: # recognize a new click
data.resetSliderButtonClicks = clicks
return (0, 100)
if not selected or not resultName:
return (0, 100)
minX, maxX = selected['range']['x']
values = data.getOutValues(simId, scenario, resultName)
minQ = stats.percentileofscore(values, minX)
maxQ = stats.percentileofscore(values, maxX)
return [minQ, maxQ]
# scatterplot matrix
@app.callback(Output('scatter-matrix', 'figure'),
[Input('scatterplot-button', 'n_clicks')], [
State('sim-chooser', 'value'),
State('scenario-chooser', 'value'),
State('multi-output-chooser', 'values'),
State('input-chooser', 'value')
])
def showScatterMatrix(nclicks, simId, scenario, outputs, inputs):
inputs = inputs or data.paraCoordsVars
#outputs = outputs or data.getOutputsWithValues(simId, scenario)
_logger.debug('showScatterMatrix(%s, %s, %s, %s)' %
(simId, scenario, inputs, outputs))
if not inputs or not outputs:
return ''
figure = data.scatterPlots(simId, scenario, inputs, outputs)
return figure
if True:
# TBD: not needed in all cases; restore this as option later
# correlation convergence plot
@app.callback(Output('corr-convergence', 'figure'), [
Input('project-chooser', 'value'),
Input('sim-chooser', 'value'),
Input('scenario-chooser', 'value'),
Input('output-chooser', 'value')
])
def showCorrConvergence(project, simId, scenario, resultName):
if not (project and scenario and resultName) or simId is None:
return ''
figure = data.corrConvergencePlot(simId, scenario, resultName)
return figure
app.run_server(debug=args.debug,
threaded=False,
host=args.host,
port=args.port)
def addArgs(self, parser):
from pygcam.config import getParam, getParamAsFloat, getParamAsInt
defaultProfile = getParam('IPP.Profile')
defaultClusterId = getParam('IPP.ClusterId')
defaultQueue = getParam('IPP.Queue')
defaultWorkDir = getParam('IPP.WorkDir')
defaultStopJobsCmd = getParam('IPP.StopJobsCommand')
defaultMaxEngines = getParamAsInt('IPP.MaxEngines')
defaultMinutes = getParamAsFloat('IPP.MinutesPerRun')
parser.add_argument('mode',
choices=['start', 'stop'],
help='''Whether to start or stop the cluster''')
parser.add_argument(
'-c',
'--clusterId',
type=str,
default=defaultClusterId,
help='''A string to identify this cluster. Default is the
value of config var IPP.ClusterId, currently
"%s".''' % defaultClusterId)
parser.add_argument('-e',
'--maxEngines',
type=int,
default=defaultMaxEngines,
help='''Set maximum number of engines to create.
Overrides config parameter IPP.MaxEngines, currently
%s''' % defaultMaxEngines)
parser.add_argument(
'-m',
'--minutesPerRun',
type=int,
default=defaultMinutes,
help='''Set the number of minutes of walltime to allocate
per GCAM run. Overrides config parameter IPP.MinutesPerRun,
currently %s.''' % defaultMinutes)
parser.add_argument(
'-n',
'--numTrials',
type=int,
default=10,
help='''The total number of GCAM trials that will be run on this
cluster. (Relevant only for "start" command.)''')
parser.add_argument(
'-o',
'--otherArgs',
type=str,
default='',
help='Command line arguments to append to the ipcluster command.')
parser.add_argument(
'-p',
'--profile',
type=str,
default=defaultProfile,
help='''The name of the ipython profile to use. Default is
the value of config var IPP.Profile, currently
"%s".''' % defaultProfile)
parser.add_argument(
'-q',
'--queue',
type=str,
default=defaultQueue,
help='''The queue or partition on which to create the controller
and engines. Overrides config var IPP.Queue, currently
"%s".''' % defaultQueue)
parser.add_argument(
'-s',
'--stopJobs',
action='store_true',
help='''Stop running jobs using the value if IPP.StopJobsCommand,
currently "%s". (Ignored for mode "start".)''' %
defaultStopJobsCmd)
parser.add_argument(
'-w',
'--workDir',
type=str,
default=defaultWorkDir,
help='''Where to run the ipcluster command. Overrides the
value of config var IPP.WorkDir, currently '%s'.'''
% defaultWorkDir)
return parser # for auto-doc generation
def createScenario(inputFile, xmlFile, trialDir):
basedir = pathjoin(trialDir, 'trial-xml', 'local-xml', GROUP, BASELINE)
valuesFile = pathjoin(basedir, 'mcsValues.xml')
# This requires that mcsValues.xml is updated before the func is called, so
# the <InputFile> specifying the "writeFunc" must come after mcsValues.xml.
mcsValues = McsValues(valuesFile)
value_names = {
'pop': None, # "pop" values are 2 and 5, used directly
'dds':
['A2',
'constant'], # for other params, values are 0 or 1, used as indices
'cons_dem': ['ref', 'high'],
'ccs': ['ref', 'no'],
're': ['ref', 'adv'],
'bev': ['ref', 'adv'],
'nuc_ret': ['ref', 'fast']
}
def value_by_name(param):
value = int(
mcsValues.valueForRegion(param, 'global')
) # all values are int, though MCS system renders as floats
names = value_names[param]
return value if names is None else names[
value] # use trial value as index into names list
# copy the trial values for these parameters into a dictionary keyed by param name
v = {param: value_by_name(param) for param in value_names.keys()}
pop = v['pop']
dds = v['dds']
ccs = v['ccs']
re = v['re']
bev = v['bev']
cons_dem = v['cons_dem']
nuc_ret = v['nuc_ret']
# construct a mnemonic scenario name
scenario = f"SSP{pop}-CI{dds}-CD{cons_dem}-CCS{ccs}-RE{re}-BEV{bev}-NUC{nuc_ret}"
_logger.info("(writeFunc) running scenario '%s'", scenario)
# tree = xmlFile.tree # useful IFF tree refers to config.xml
from pygcam.mcs.XMLConfigFile import XMLConfigFile, COMPONENTS_GROUP
# from pygcam.xmlEditor import xmlEdit
# context = 'whatever'
# cfg = XMLConfigFile.getConfigForScenario(context, useCopy=False)
# Or this?
from pygcam.xmlSetup import scenarioEditor
xmlSrcDir = getParam(
'GCAM.XmlSrc') # or is it a subdir of this that we need?
refWorkspace = getParam('MCS.RunWorkspace')
ed = scenarioEditor('mcs',
baseline='mcs',
xmlSrcDir=xmlSrcDir,
refWorkspace=refWorkspace,
mcsMode='trial')
cfg_path = ed.cfgPath()
print(f"writeFuncs.py: config path is {cfg_path}")
# ed.addScenarioComponent(name, pathname)
# cfg.insertComponentPathname(name, pathname, after)
# cfg.updateComponentPathname(name, pathname)
# cfg.deleteComponent(name)
# <add name="SSP">../input/policy/scenarios/pop_gdp_{pop}.xml</add>
ed.addScenarioComponent("SSP",
f"../input/policy/scenarios/pop_gdp_{pop}.xml")
# <replace name="dd_usa">../input/policy/scenarios/HDDCDD_{dds}.xml</replace>
ed.updateScenarioComponent("dd_usa",
f"../input/policy/scenarios/HDDCDD_{dds}.xml")
# <add name="trn_pe_zero">../input/policy/scenarios/trn_pe_zero.xml</add>
# <add name="trn_dem">../input/policy/scenarios/td_{pop}_{cons_dem}.xml</add>
# <add name="FLSP">../input/policy/scenarios/FLSP_{pop}_{cons_dem}.xml</add>
ed.addScenarioComponent("trn_pe_zero",
"../input/policy/scenarios/trn_pe_zero.xml")
ed.addScenarioComponent(
"trn_dem", f"../input/policy/scenarios/td_{pop}_{cons_dem}.xml")
ed.addScenarioComponent(
"FLSP", f"../input/policy/scenarios/FLSP_{pop}_{cons_dem}.xml")
# <if value1="{ccs}" value2="no">
# <add name="noccs_st">../input/policy/scenarios/noccs_st.xml</add>
# <add name="noccs_pt">../input/policy/scenarios/noccs_pt.xml</add>
# </if>
if ccs == "no":
ed.addScenarioComponent("noccs_st",
"../input/policy/scenarios/noccs_st.xml")
ed.addScenarioComponent("noccs_pt",
"../input/policy/scenarios/noccs_pt.xml")
# <if value1="{re}" value2="adv">
# <add name="re_adv">../input/policy/scenarios/adv_re_tech.xml</add>
# </if>
if re == "adv":
ed.addScenarioComponent("re_adv",
"../input/policy/scenarios/adv_re_tech.xml")
# <if value1="{bev}" value2="adv">
# <replace name="bev_UCD">../input/policy/scenarios/bev_UCD_adv.xml</replace>
# <insert name="bev_USA_adv" after="bev_USA">../input/policy/scenarios/bev_USA_adv.xml</insert>
# <add name="BEV_LoBatt_compact">../input/policy/reference/BEV_LoBatt_compact_wLearning.xml</add>
# <add name="BEV_LoBatt_large">../input/policy/reference/BEV_LoBatt_large_wLearning.xml</add>
# <add name="BEV_LoBatt_lttruck_SUV">../input/policy/reference/BEV_LoBatt_lttruck_SUV_wLearning.xml</add>
# <add name="BEV_LoBatt_midsize">../input/policy/reference/BEV_LoBatt_midsize_wLearning.xml</add>
# <add name="BEV_LDV_SW">../input/policy/reference/BEV_SW_update-global.xml</add>
# </if>
if bev == "adv":
ed.updateScenarioComponent(
"bev_UCD", "../input/policy/scenarios/bev_UCD_adv.xml")
ed.insertScenarioComponent(
"bev_USA_adv", "../input/policy/scenarios/bev_USA_adv.xml",
"bev_USA")
ed.addScenarioComponent(
"BEV_LoBatt_compact",
"../input/policy/reference/BEV_LoBatt_compact_wLearning.xml")
ed.addScenarioComponent(
"BEV_LoBatt_large",
"../input/policy/reference/BEV_LoBatt_large_wLearning.xml")
ed.addScenarioComponent(
"BEV_LoBatt_lttruck_SUV",
"../input/policy/reference/BEV_LoBatt_lttruck_SUV_wLearning.xml")
ed.addScenarioComponent(
"BEV_LoBatt_midsize",
"../input/policy/reference/BEV_LoBatt_midsize_wLearning.xml")
ed.addScenarioComponent(
"BEV_LDV_SW", "../input/policy/reference/BEV_SW_update-global.xml")
# <if value1="{nuc_ret}" value2="fast">
# <function name="multiply">tag="nuc_usa",xpath="//half-life",value=0.6</function>
# <add name="nuc_bound">../input/policy/scenarios/nonewnuclear.xml</add>
# </if>
if nuc_ret == "fast":
# <function name="multiply">tag="nuc_usa",xpath="//half-life",value=0.6</function>
ed.multiply("nuc_usa", "//half-life", 0.6)
ed.addScenarioComponent("nuc_bound",
"../input/policy/scenarios/nonewnuclear.xml")
cfg.write(path=configPath)
def addArgs(self, parser):
parser.add_argument(
'--delete',
action='store_true',
help='''DELETE and recreate the simulation "run" directory.''')
parser.add_argument(
'-D',
'--desc',
type=str,
default='',
help='A brief (<= 256 char) description the simulation.')
parser.add_argument(
'-e',
'--exportVars',
default='',
help=
'Export variable and distribution info in a tab-delimited file with the given name and exit.'
)
parser.add_argument(
'-g',
'--groupName',
default='',
help='''The name of a scenario group to process.''')
parser.add_argument(
'-m',
'--method',
choices=['montecarlo', 'sobol', 'fast', 'morris'],
default='montecarlo',
help=
'''Use the specified method to generate trial data. Default is "montecarlo".'''
)
parser.add_argument(
'-o',
'--outFile',
help='''For methods other than "montecarlo". The path to a "package
directory" into which SALib-related data are stored.
If the filename does not end in '.sa', this extension is added. The file
'problem.csv' within the package directory will contain the parameter specs in
SALib format. The file inputs.csv is also generated in the file package using
the chosen method's sampling method. If an outFile is not specified, a package
of the name 'data.sa' is created in the simulation run-time directory.'''
)
parser.add_argument(
'-p',
'--paramFile',
default=None,
help='''Specify an XML file containing parameter definitions.
Defaults to the value of config parameter MCS.ParametersFile
(currently %s)''' % getParam('MCS.ParametersFile'))
runRoot = getParam('MCS.Root')
parser.add_argument(
'-r',
'--runRoot',
default=None,
help=
'''Root of the run-time directory for running user programs. Defaults to
value of config parameter MCS.Root (currently %s)'''
% runRoot)
parser.add_argument(
'-S',
'--calcSecondOrder',
action='store_true',
help=
'''For Sobol method only -- calculate second-order sensitivities.'''
)
parser.add_argument('-s',
'--simId',
type=int,
default=1,
help='The id of the simulation. Default is 1.')
parser.add_argument(
'-t',
'--trials',
type=int,
default=-1,
help=
'''The number of trials to create for this simulation (REQUIRED). If a
value of 0 is given, scenario setup is performed, scenario names are added to
the database, and meta-data is copied, but new trial data is not generated.'''
)
return parser # for auto-doc generation
def defaultGroupName(cls):
from pygcam.config import getParam
projectName = getParam('GCAM.ProjectName')
obj = cls.readProjectFile(projectName)
return obj.scenarioSetup.defaultGroup
def genSimulation(simId, trials, paramPath, args):
'''
Generate a simulation based on the given parameters.
'''
from ..context import Context
from ..Database import getDatabase
from ..XMLParameterFile import XMLParameterFile
from ..util import getSimParameterFile, getSimResultFile, symlink, filecopy
from pygcam.constants import LOCAL_XML_NAME
from pygcam.project import Project
from pygcam.xmlSetup import ScenarioSetup
runInputDir = getParam('MCS.RunInputDir')
runWorkspace = getParam('MCS.RunWorkspace')
# Add symlink to workspace's input dir so we can find XML files using rel paths in config files
simDir = getSimDir(simId, create=True)
simInputDir = os.path.join(simDir, 'input')
symlink(runInputDir, simInputDir)
# Ditto for workspace's local-xml
workspaceLocalXml = os.path.join(runWorkspace, LOCAL_XML_NAME)
simLocalXmlDir = os.path.join(simDir, LOCAL_XML_NAME)
symlink(workspaceLocalXml, simLocalXmlDir)
projectName = getParam('GCAM.ProjectName')
project = Project.readProjectFile(projectName, groupName=args.groupName)
args.groupName = groupName = args.groupName or project.scenarioSetup.defaultGroup
# Run static setup for all scenarios in the given group
runStaticSetup(runWorkspace, project, groupName)
# TBD: Use pygcam scenario def and copy pygcam files, too
scenarioFile = getParam('GCAM.ScenarioSetupFile')
scenarioSetup = ScenarioSetup.parse(scenarioFile)
scenarioNames = scenarioSetup.scenariosInGroup(groupName)
baseline = scenarioSetup.baselineForGroup(groupName)
# Copy the user's results.xml file to {simDir}/app-xml
userResultFile = getParam('MCS.ResultsFile')
simResultFile = getSimResultFile(simId)
mkdirs(os.path.dirname(simResultFile))
filecopy(userResultFile, simResultFile)
paramFileObj = XMLParameterFile(paramPath)
context = Context(projectName=args.projectName,
simId=simId,
groupName=groupName)
paramFileObj.loadInputFiles(context, scenarioNames, writeConfigFiles=True)
# Define the experiments (scenarios) in the database
db = getDatabase()
db.addExperiments(scenarioNames, baseline, scenarioFile)
if not trials:
_logger.warn("Simulation meta-data has been copied.")
return
paramFileObj.generateRandomVars()
_logger.info("Generating %d trials to %r", trials, simDir)
df = genTrialData(simId, trials, paramFileObj, args)
# Save generated values to the database for post-processing
saveTrialData(df, simId)
# Also save the param file as parameters.xml, for reference only
simParamFile = getSimParameterFile(simId)
filecopy(paramPath, simParamFile)
