def makePlotPath(value, simId): plotDir = getParam('MCS.PlotDir') subDir = os.path.join(plotDir, "s%d" % simId) mkdirs(subDir) plotType = getParam('MCS.PlotType') path = os.path.join(subDir, "%s.%s" % (value, plotType)) #print "Plot path: ", path return path
def addArgs(self, parser): from pygcam.config import getParam #, getParamAsInt # defaultQueue = getParam('IPP.Queue') defaultProfile = getParam('IPP.Profile') defaultClusterId = getParam('IPP.ClusterId') defaultWorkDir = getParam('IPP.WorkDir') defaultNumTrials = 1 parser.add_argument( '-c', '--clusterId', type=str, default=defaultClusterId, help='''A string to identify this cluster. Default is the value of config var IPP.ClusterId, currently "%s".''' % defaultClusterId) parser.add_argument( '-n', '--numTrials', type=int, default=defaultNumTrials, help='''The number of additional trials to create engines for. Default is %d''' % defaultNumTrials) parser.add_argument( '-o', '--otherArgs', type=str, default='', help='Command line arguments to append to the ipengine command.') parser.add_argument( '-p', '--profile', type=str, default=defaultProfile, help='''The name of the ipython profile to use. Default is the value of config var IPP.Profile, currently "%s".''' % defaultProfile) # parser.add_argument('-q', '--queue', type=str, default=defaultQueue, # help='''The queue or partition on which to create the controller # and engines. Overrides config var IPP.Queue, currently # "%s".''' % defaultQueue) parser.add_argument( '-w', '--workDir', type=str, default=defaultWorkDir, help='''Where to run the ipengine command. Overrides the value of config var IPP.WorkDir, currently '%s'.''' % defaultWorkDir) return parser # for auto-doc generation
def driver(args, tool): ''' Generate a simulation. Do generic setup, then call genSimulation(). ''' from pygcam.utils import removeTreeSafely from ..Database import getDatabase from ..error import PygcamMcsUserError from ..util import saveDict paramFile = args.paramFile or getParam('MCS.ParametersFile') if args.exportVars: _exportVars(paramFile, args.exportVars) return simId = args.simId desc = args.desc trials = args.trials if trials < 0: raise PygcamMcsUserError( "Trials argument is required: must be an integer >= 0") projectName = args.projectName runRoot = args.runRoot if runRoot: # TBD: write this to config file under [project] section setParam('MCS.Root', runRoot, section=projectName) _logger.info( 'Please add "MCS.Root = %s" to your .pygcam.cfg file in the [%s] section.', runRoot, projectName) runDir = getParam('MCS.RunDir', section=projectName) if args.delete: removeTreeSafely(runDir, ignore_errors=False) runWorkspace = getParam('MCS.RunWorkspace') if not os.path.exists(runWorkspace): _newsim(runWorkspace, trials) # Called with trials == 0 when setting up a local run directory on /scratch if trials: # The simId can be provided on command line, in which case we need # to delete existing parameter entries for this app and simId. db = getDatabase() simId = db.createSim(trials, desc, simId=simId) genSimulation(simId, trials, paramFile, args=args) if trials: # Save a copy of the arguments used to create this simulation simDir = getSimDir(simId) argSaveFile = '%s/gcamGenSimArgs.txt' % simDir saveDict(vars(args), argSaveFile)
def __init__(self, runId=None, projectName=None, simId=None, trialNum=None, scenario=None, baseline=None, groupName=None, status=None, store=True): self.runId = runId self.simId = simId self.trialNum = trialNum self.scenario = scenario self.baseline = baseline self.status = status self.projectName = projectName = projectName or getParam( 'GCAM.DefaultProject') project = Project.readProjectFile(projectName) self.groupName = groupName or project.scenarioSetup.defaultGroup self.useGroupDir = project.scenarioGroup.useGroupDir if store and runId: self.saveRunInfo()
def _runPygcamSteps(steps, context, runWorkspace=None, raiseError=True): """ run "gt +P {project} --mcs=trial run -s {step[,step,...]} -S {scenarioName} ..." For Monte Carlo trials. """ import pygcam.tool runWorkspace = runWorkspace or getParam('MCS.RunWorkspace') trialDir = context.getTrialDir() groupArg = ['-g', context.groupName] if context.groupName else [] # N.B. MCS.RunWorkspace is the RefWorkspace for trial sandboxes toolArgs = [ '+P', context.projectName, '--mcs=trial', '--set=GCAM.SandboxRefWorkspace=' + runWorkspace, 'run', '-s', steps, '-S', context.scenario, '--sandboxDir=' + trialDir ] + groupArg command = 'gt ' + ' '.join(toolArgs) _logger.debug('Running: %s', command) status = pygcam.tool.main(argv=toolArgs, raiseError=True) msg = '"%s" exited with status %d' % (command, status) if status != 0 and raiseError: raise GcamToolError(msg) _logger.info("_runSteps: " + msg) return status
def driver(args, tool): import os ''' Set up the SQL database. Must call startDb() before calling this. ''' import shutil from pygcam.config import getParam from ..Database import getDatabase from ..error import PygcamMcsSystemError if args.deleteSims: # Remove the whole sims dir and remake it runSimsDir = getParam('MCS.RunSimsDir') if os.path.exists(runSimsDir): try: shutil.rmtree(runSimsDir) os.mkdir(runSimsDir) except Exception as e: raise PygcamMcsSystemError( 'Failed to recreate sim dir %s: %s' % (runSimsDir, e)) db = getDatabase() # reinitialize the db db.initDb(args=args) if not (args.empty): from ..XMLResultFile import XMLResultFile XMLResultFile.addOutputs()
def _newsim(runWorkspace, trials): ''' Setup the app and run directories for a given user app. ''' from pygcam.scenarioSetup import copyWorkspace from ..Database import getDatabase from ..error import PygcamMcsUserError from ..XMLResultFile import XMLResultFile if not runWorkspace: raise PygcamMcsUserError( "MCS.RunWorkspace was not set in the configuration file") srcDir = getParam('GCAM.RefWorkspace') dstDir = runWorkspace copyWorkspace(dstDir, refWorkspace=srcDir, forceCreate=True, mcsMode=True) if trials: db = getDatabase() # ensures database initialization XMLResultFile.addOutputs() # Load SQL script to create convenient views text = getResource('mcs/etc/views.sql') db.executeScript(text=text)
def projectChooser(self): layout = dcc.Dropdown(id='project-chooser', options=[{ 'label': name, 'value': name } for name in self.projects], value=getParam('GCAM.DefaultProject')) return layout
def getRunQueryDir(): """ Returns the path to sim's copy of the scenarios.xml file. """ workspace = getParam('MCS.RunWorkspace') path = os.path.join(workspace, QueryDirName) return path
def readProjectFile(cls, projectName, groupName=None, projectFile=None): # return cached project if already read, otherwise read project.xml if not cls.instance or cls.instance.projectName != projectName: projectFile = projectFile or getParam('GCAM.ProjectXmlFile', section=projectName) cls.instance = Project(projectFile, projectName, groupName) return cls.instance
def projectsWithDatabases(): withDatabases = [] for project in getSections(): dbPath = getParam('MCS.DbPath', section=project, raiseError=False) if dbPath and os.path.exists(dbPath): withDatabases.append(project) return withDatabases
def _readParameterInfo(context, paramPath): from pygcam.xmlSetup import ScenarioSetup scenarioFile = getParam('GCAM.ScenarioSetupFile') scenarioSetup = ScenarioSetup.parse(scenarioFile) scenarioNames = scenarioSetup.scenariosInGroup(context.groupName) paramFile = XMLParameterFile(paramPath) paramFile.loadInputFiles(context, scenarioNames, writeConfigFiles=False) paramFile.runQueries() return paramFile
def driver(args, tool): """ Start ipyparallel engines. """ from pygcam.config import getParam from ..master import startEngines, templatePath scheduler = getParam('IPP.Scheduler') batchTemplate = templatePath(scheduler, args.profile, args.clusterId, 'engine') startEngines(args.numTrials, batchTemplate)
def _setup_werkzeug_log(level, host, port): import logging consoleFormat = getParam('GCAM.LogConsoleFormat') handler = logging.StreamHandler() handler.setFormatter(logging.Formatter(consoleFormat)) log = logging.getLogger('werkzeug') log.addHandler(handler) log.setLevel(level) log.propagate = False # Setting Flask.LogLevel > INFO quashes this useful message, so we restore it if log.level > logging.INFO: print('* Running on http://{}:{}/ (Press CTRL+C to quit)'.format( host, port))
def createOutputDir(outputDir): from ..utils import removeFileOrTree from ..temp_file import getTempDir removeFileOrTree(outputDir, raiseError=False) tempOutputDir = getParam('MCS.TempOutputDir') if tempOutputDir: # We create this on /scratch which is purged automatically. newDir = getTempDir(suffix='', tmpDir=tempOutputDir, delete=False) mkdirs(newDir) _logger.debug("Creating '%s' link to %s" % (outputDir, newDir)) symlink(newDir, outputDir) else: mkdirs(outputDir)
def getSimDir(simId, create=False): ''' Return and optionally create the path to the top-level simulation directory for the given simulation number, based on the SimsDir parameter specified in the config file. ''' simsDir = getParam('MCS.RunSimsDir') if not simsDir: raise PygcamMcsUserError( "Missing required config parameter 'RunSimsDir'") simDir = os.path.join(simsDir, 's%03d' % simId) # name is of format ".../s001/" if create: mkdirs(simDir) return simDir
def run(self, args, tool): # If not passed on command-line, read from config file yearsStr = args.years or getParam('GCAM.Years') years = [int(s) for s in yearsStr.split('-')] if len(years) != 2: raise PygcamException('''Years must be specified as XXXX-YYYY, where XXXX and YYYY are the first and last years to consider, respectively''') firstYear, lastYear = years guess = shockedFuel(args.policy) _logger.info("Inferred shocked fuel '%s' from policy '%s'", guess, args.policy) shocked = [guess] GcamResultProcessor(args.baseline, args.policy, args.diffsDir, shocked, args.fuelShockFile, firstYear=firstYear, lastYear=lastYear)
def main(): import pandas as pd ref_ws = getParam('GCAM.RefWorkspace') data_sys = pathjoin(ref_ws, 'input', 'gcam-data-system') aglu_xml = pathjoin(data_sys, 'xml', 'aglu-xml') protected_land_2_xml = pathjoin(aglu_xml, 'protected_land_input_2.xml') protected_land_3_xml = pathjoin(aglu_xml, 'protected_land_input_3.xml') parser = ET.XMLParser(remove_blank_text=True, remove_comments=True) prot2 = ET.parse(protected_land_2_xml, parser) prot3 = ET.parse(protected_land_3_xml, parser) regionMap = pd.read_csv(pathjoin(data_sys, '_common', 'mappings', 'GCAM_region_names_32reg.csv'), skiprows=3) regionMap.set_index('GCAM_region_ID', inplace=True) region = regionMap.region # Read the CSV defining the region-wide land protection levels for OTAQ project and # apply this anew, verifying that the only differences are way into the decimal places otaq_prot_file = pathjoin(ref_ws, 'input', 'otaq-modifications', 'aglu', 'OTAQ_land_protection.csv') otaq_prot_df = pd.read_csv(otaq_prot_file) otaq_prot_df['region'] = otaq_prot_df['GCAM_region_ID'].map(region) otaq_prot_df.drop('GCAM_region_ID', axis=1, inplace=True) otaq_prot_df.set_index('region', inplace=True) fracs = otaq_prot_df.protect_land_fract # protected_land_2 has only UnmanagedPasture prot2_dict = {reg: [('UnmanagedPasture', fracs[reg])] for reg in fracs.index} _protect_land(prot2, prot2_dict) outfile = "/Users/rjp/Downloads/land_prot2_modified.xml" print("Writing", outfile) prot2.write(outfile, xml_declaration=True, pretty_print=True) # protected_land_3 has all the other unmanaged types landtypes = ['UnmanagedForest', 'Grassland', 'Shrubland'] prot3_dict = {reg: [(landtype, fracs[reg]) for landtype in landtypes] for reg in fracs.index} _protect_land(prot3, prot3_dict) outfile = "/Users/rjp/Downloads/land_prot3_modified.xml" print("Writing", outfile) prot3.write(outfile, xml_declaration=True, pretty_print=True)
def main(): args = parseArgs() collection_file = Path(args.collection_file) if collection_file.exists(): collection_file.rename(collection_file.name + '~') wrapper_file = args.wrapper_file sandbox_dir = Path(args.sandbox_dir or getParam('GCAM.SandboxDir')) with open(collection_file, 'w') as output: paths = sandbox_dir.glob(f'*/exe/{wrapper_file}') for p in paths: scenario = p.parent.parent.name with open(p, 'r') as input: lines = input.readlines() for line in lines: output.write(f'{scenario},{line}')
def __init__(self, filename, load=True, schemaPath=None, removeComments=True, conditionalXML=False, varDict=None): self.filename = filename self.tree = None self.conditionalXML = conditionalXML self.varDict = varDict or getConfigDict( section=getParam('GCAM.DefaultProject')) self.removeComments = removeComments self.schemaPath = schemaPath self.schemaStream = None if filename and load: self.read()
def readMetaData(self, project=None): if project: setParam('GCAM.DefaultProject', project, DEFAULT_SECTION) setSection(project) else: project = getParam('GCAM.DefaultProject') self.project = project self.inputsDF = None self.resultDict = None self.selectedResults = None self.corrDF = None self.simId = None if self.db: self.db.close() db = self.db = getDatabase() self.inputs = db.getInputs()
def activeYears(asInt=False): ''' Convert a string identifying active years into a list of ints or strs. Values must be comma-separated integers or expressions of the form xxxx-yyyy or xxxx-yyyy:z. The two expressions indicate ranges of years, with a default timestep of 5 years. If given, the final value after the colon indicates an alternative timestep. ''' global _activeYearInts global _activeYearStrs # return cached values or compute and cache result if not _activeYearStrs: import re from functools import reduce def reducer(lst, item): m = re.match(r"^(\d{4})-(\d{4})(?::(\d+))?$", item) if m: start = int(m.group(1)) end = int(m.group(2)) step = int(m.group( 3)) if m.lastindex == 3 else 5 # default 5 yr timestep rng = list(range(start, end + 1, step)) lst.extend(rng) elif item.isdigit(): lst.append(int(item)) else: raise PygcamMcsUserError( 'Element in list of active years is not an integer: %s' % item) return lst yearStr = getParam('MCS.Years') items = yearStr.split(',') years = reduce(reducer, items, []) _activeYearInts = [int(y) for y in years] _activeYearStrs = [str(y) for y in years] return _activeYearInts if asInt else _activeYearStrs
def parseArgs(): parser = argparse.ArgumentParser( description= '''Collect files written by custom GCAM wrapper into a single file with each line containing the name of scenario generating the file.''' ) wrapper_file = './failed_periods.txt' parser.add_argument( '-f', '--wrapper_file', default=wrapper_file, help= f'''The name of the file created by the custom GCAM wrapper function. Default is '{wrapper_file}'.''') collection_file = './collected_wrapper_output.txt' parser.add_argument( '-o', '--collection_file', default=collection_file, help=f'''The directory into which all normalized XML files are written. Default is '{collection_file}'.''') sandbox_dir = getParam('GCAM.SandboxDir') parser.add_argument( '-s', '--sandbox_dir', default=None, help= f'''The directory containing folders for the scenarios from which the wrapper output files should be collected. Default is taken from pygcam config variable "GCAM.SandboxDir", currently "{sandbox_dir}".''') args = parser.parse_args() return args
def driver(args, tool): """ Analyze MCS results """ import os from ..analysis import analyzeSimulation from ..error import PygcamMcsUserError if args.timeseries: import pandas as pd from pygcam.config import getParam from ..Database import getDatabase from ..timeseriesPlot import plotTimeSeries, plotForcingSubplots from ..util import stripYearPrefix simId = args.simId expList = args.expName.split(',') resultName = args.resultName xlabel = 'Year' # args.xlabel or 'Year' ymin = args.ymin ymax = args.ymax # special purpose plot forcingPlot = args.forcingPlot if not (expList[0] and resultName): raise PygcamMcsUserError( "expName and resultName must be specified") db = getDatabase() trialCount = db.getTrialCount(simId) plotDir = getParam('MCS.PlotDir') plotType = getParam('MCS.PlotType') allResults = db.getTimeSeries(simId, resultName, expList) # , regionName) if not allResults: raise PygcamMcsUserError('No timeseries results for simId=%d, expList=%s, resultName=%s' \ % (simId, expList, resultName)) def computeFilename(expName): basename = "%s-s%d-%s.%s" % (resultName, simId, expName, plotType) filename = os.path.join(plotDir, 's%d' % simId, basename) return filename # massage the data into the format required by plotTimeSeries def createRecord(pair): obj, expName = pair d = obj.__dict__ d['expName'] = expName return d records = [createRecord(pair) for pair in allResults] resultDF = pd.DataFrame.from_records(records, index='seriesId') units = resultDF.units.iloc[0] resultDF.drop(['units', '_sa_instance_state', 'outputId'], axis=1, inplace=True) # convert column names like 'y2020' to '2020' cols = [stripYearPrefix(c) for c in resultDF.columns] resultDF.columns = cols if forcingPlot: filename = computeFilename('combo') plotForcingSubplots(resultDF, filename=filename, ci=95, show_figure=False) return for expName in expList: # create a copy so we can drop expName column for melt df = resultDF.query("expName == '%s'" % expName).copy() _logger.debug("Found %d result records for exp %s, result %s" % (len(df), expName, resultName)) df.drop(['expName'], axis=1, inplace=True) df = pd.melt(df, id_vars=['runId'], var_name='year') title = '%s for %s' % (resultName, expName) filename = computeFilename(expName) _logger.debug('Saving timeseries plot to %s' % filename) reg = "" # '-' + regionName if regionName else "" extra = "name=%s trials=%d/%d simId=%d scenario=%s%s" % \ (resultName, resultDF.shape[0], trialCount, simId, expName, reg) # TBD: generalize this with a lookup table or file if units == 'W/m^2': units = 'W m$^{-2}$' plotTimeSeries( df, 'year', 'runId', title=title, xlabel=xlabel, ylabel=units, ci=[95], # [50, 95], text_label=None, legend_name=None, legend_labels=None, ymin=ymin, ymax=ymax, filename=filename, show_figure=False, extra=extra) # if doing timeseries plot, none of the other options are relevant return if not (args.exportInputs or args.resultFile or args.plot or args.importance or args.groups or args.plotInputs or args.stats or args.convergence or args.exportEMA, args.exportAll): msg = 'Must specify at least one of: --export, --resultFile, --plot, --importance, --groups, --distros, --stats, --convergence, --exportEMA, --exportAll' raise PygcamMcsUserError(msg) analyzeSimulation(args)
def main(args): app = dash.Dash(name='mcs-explorer', csrf_protect=False) level = getParam('Flask.LogLevel') flaskLog = app.server.logger flaskLog.setLevel(level) _setup_werkzeug_log(level, args.host, args.port) # app.config.supress_callback_exceptions = True app.css.append_css({ "external_url": "https://fonts.googleapis.com/css?family=Dosis:regular,bold|Lato" }) app.css.append_css( {"external_url": "https://codepen.io/plevin/pen/MvpeNV.css"}) data = McsData(app) data.readMetaData() app.layout = data.layout() # # Callbacks # generateDropdownDefaults(app, ['scenario-chooser', 'output-chooser']) @app.callback(Output('output-chooser', 'options'), [ Input('project-chooser', 'value'), Input('sim-chooser', 'value'), Input('scenario-chooser', 'value') ]) def updateOutputChooser(project, simId, scenario): _logger.debug('updateOutputChooser(%s, %s, %s)' % (project, simId, scenario)) layout = data.outputChooser(simId=simId, scenario=scenario, optionsOnly=True) return layout @app.callback(Output('multi-output-chooser', 'options'), [ Input('project-chooser', 'value'), Input('sim-chooser', 'value'), Input('scenario-chooser', 'value') ]) def updateMultiOutputChooser(project, simId, scenario): _logger.debug('updateMultiOutputChooser(%s, %s, %s)' % (project, simId, scenario)) layout = data.multiOutputChooser(simId=simId, scenario=scenario, optionsOnly=True) return layout @app.callback( Output('scenario-chooser', 'options'), [Input('project-chooser', 'value'), Input('sim-chooser', 'value')]) def updateScenarioChooser(project, simId): _logger.debug('updateScenarioChooser(%s, %s)' % (project, simId)) layout = data.scenarioChooser(simId, multi=False, optionsOnly=True) return layout @app.callback(Output('sim-chooser', 'options'), [Input('project-chooser', 'value')]) def updateSimChooser(project): _logger.debug('updateSimChooser(%s)' % project) data.readMetaData(project) layout = data.simChooser(optionsOnly=True) return layout # TBD: finish writing these methods to store data in hidden div for # TBD: reset button and selection to simplify callback for drawing dist # # The next few callbacks use a hidden <div> to store # communicate the variables used in the parallel coords # figure with those in the output chooser. # @app.callback(Output('paraCoords-vars', 'children'), [Input('paraCoords', 'figure')]) def updateParaCoordsVars(figure): varNames = data.paraCoordsVars _logger.debug('updateParaCoordsVars vars=%s' % varNames) varNamesJSON = json.dumps(varNames) return varNamesJSON @app.callback(Output('input-chooser', 'value'), [Input('paraCoords-vars', 'children')]) def updateInputChooser(varNamesJSON): varNames = json.loads(varNamesJSON) _logger.debug('updateInputChooser(%s)' % varNames) return varNames @app.callback( Output('paraCoords', 'figure'), [ Input('project-chooser', 'value'), Input('sim-chooser', 'value'), Input('scenario-chooser', 'value'), Input('output-chooser', 'value'), Input('paraCoords-slider', 'value'), Input('distribution', 'figure'), # force this to plot last ]) def showParallelCoords(project, simId, scenario, resultName, varsToShow, figure): return data.parcoordsPlot(simId, scenario, resultName, varsToShow) @app.callback(Output('percentile-text', 'children'), [Input('dist-slider', 'value')]) def showDistSliderValues(sliderInfo): return '%d%%-%d%%' % tuple(sliderInfo) @app.callback(Output('tornado', 'figure'), [ Input('project-chooser', 'value'), Input('sim-chooser', 'value'), Input('scenario-chooser', 'value'), Input('output-chooser', 'value'), Input('tornado-slider', 'value'), Input('tornado-type-chooser', 'value'), Input('distribution', 'selectedData'), ]) def showTornado(project, simId, scenario, resultName, sliderValue, tornadoType, selectedData): _logger.debug('showTornado(%s, %s, %s, %s, %s, %s, %s)' % (project, simId, scenario, resultName, sliderValue, tornadoType, selectedData)) if not (project and scenario and resultName) or simId is None: return '' figure = data.tornadoPlot(simId, scenario, resultName, tornadoType, sliderValue, selectedData) return figure @app.callback(Output('distribution', 'figure'), [ Input('project-chooser', 'value'), Input('sim-chooser', 'value'), Input('scenario-chooser', 'value'), Input('output-chooser', 'value'), Input('dist-slider', 'value'), Input('dist-options', 'values'), Input('distribution', 'selectedData'), ]) def showDistribution(project, simId, scenario, outputName, sliderInfo, distOptions, selectedData): _logger.debug('showDistribution(%s, %s, %s, %s, %s)' % (project, simId, scenario, outputName, selectedData)) # Clear selectedData if context changes context = (project, simId, scenario, outputName) if not data.histogramContext or context != data.histogramContext: _logger.debug('showDistribution: clearing context') selectedData = sliderInfo = None data.histogramContext = context # if called before values are known, return an empty plot if not (project and scenario and outputName) or simId is None: plotData = {'x': []} title = 'Please select a model output to plot' annotations = None else: plotData, title, annotations = data.distPlot( simId, scenario, outputName, sliderInfo, distOptions, selectedData) # TBD: generalize this if Oct16: if outputName == 'percent-change': xtitle = 'Change from baseline fuel' xticksuffix = '%' else: # latex formatting is broken, but HTML works fine # xtitle = '$g CO_2 MJ^{-1}$' if Oct16 else '' xtitle = 'g CO<sub>2</sub> MJ<sup>-1</sup>' xticksuffix = None else: xtitle = data.db.getOutputUnits(outputName) xticksuffix = '' layout = updateStyle('Plot', title=title, yaxis={ 'title': 'Probability density', 'tickvals': [] }, xaxis={ 'title': xtitle, 'ticksuffix': xticksuffix, }, margin=getStyle('PlotMarginWithXTitle'), dragmode='select', showLegend=True, legend=dict(x=0.0, y=1.0, bgcolor=getColor('PlotBg'))) if annotations: layout['annotations'] = annotations figure = dict(data=plotData, layout=layout) return figure # dist -> dist slider @app.callback(Output('dist-slider', 'value'), [ Input('output-chooser', 'value'), Input('distribution', 'selectedData'), Input('dist-slider-reset-button', 'n_clicks') ], [State('sim-chooser', 'value'), State('scenario-chooser', 'value')]) def showPercentileSlider(resultName, selected, clicks, simId, scenario): _logger.debug('showPercentileSlider(%s, %s, %s)' % (resultName, selected, clicks)) if clicks != data.resetSliderButtonClicks: # recognize a new click data.resetSliderButtonClicks = clicks return (0, 100) if not selected or not resultName: return (0, 100) minX, maxX = selected['range']['x'] values = data.getOutValues(simId, scenario, resultName) minQ = stats.percentileofscore(values, minX) maxQ = stats.percentileofscore(values, maxX) return [minQ, maxQ] # scatterplot matrix @app.callback(Output('scatter-matrix', 'figure'), [Input('scatterplot-button', 'n_clicks')], [ State('sim-chooser', 'value'), State('scenario-chooser', 'value'), State('multi-output-chooser', 'values'), State('input-chooser', 'value') ]) def showScatterMatrix(nclicks, simId, scenario, outputs, inputs): inputs = inputs or data.paraCoordsVars #outputs = outputs or data.getOutputsWithValues(simId, scenario) _logger.debug('showScatterMatrix(%s, %s, %s, %s)' % (simId, scenario, inputs, outputs)) if not inputs or not outputs: return '' figure = data.scatterPlots(simId, scenario, inputs, outputs) return figure if True: # TBD: not needed in all cases; restore this as option later # correlation convergence plot @app.callback(Output('corr-convergence', 'figure'), [ Input('project-chooser', 'value'), Input('sim-chooser', 'value'), Input('scenario-chooser', 'value'), Input('output-chooser', 'value') ]) def showCorrConvergence(project, simId, scenario, resultName): if not (project and scenario and resultName) or simId is None: return '' figure = data.corrConvergencePlot(simId, scenario, resultName) return figure app.run_server(debug=args.debug, threaded=False, host=args.host, port=args.port)
def addArgs(self, parser): from pygcam.config import getParam, getParamAsFloat, getParamAsInt defaultProfile = getParam('IPP.Profile') defaultClusterId = getParam('IPP.ClusterId') defaultQueue = getParam('IPP.Queue') defaultWorkDir = getParam('IPP.WorkDir') defaultStopJobsCmd = getParam('IPP.StopJobsCommand') defaultMaxEngines = getParamAsInt('IPP.MaxEngines') defaultMinutes = getParamAsFloat('IPP.MinutesPerRun') parser.add_argument('mode', choices=['start', 'stop'], help='''Whether to start or stop the cluster''') parser.add_argument( '-c', '--clusterId', type=str, default=defaultClusterId, help='''A string to identify this cluster. Default is the value of config var IPP.ClusterId, currently "%s".''' % defaultClusterId) parser.add_argument('-e', '--maxEngines', type=int, default=defaultMaxEngines, help='''Set maximum number of engines to create. Overrides config parameter IPP.MaxEngines, currently %s''' % defaultMaxEngines) parser.add_argument( '-m', '--minutesPerRun', type=int, default=defaultMinutes, help='''Set the number of minutes of walltime to allocate per GCAM run. Overrides config parameter IPP.MinutesPerRun, currently %s.''' % defaultMinutes) parser.add_argument( '-n', '--numTrials', type=int, default=10, help='''The total number of GCAM trials that will be run on this cluster. (Relevant only for "start" command.)''') parser.add_argument( '-o', '--otherArgs', type=str, default='', help='Command line arguments to append to the ipcluster command.') parser.add_argument( '-p', '--profile', type=str, default=defaultProfile, help='''The name of the ipython profile to use. Default is the value of config var IPP.Profile, currently "%s".''' % defaultProfile) parser.add_argument( '-q', '--queue', type=str, default=defaultQueue, help='''The queue or partition on which to create the controller and engines. Overrides config var IPP.Queue, currently "%s".''' % defaultQueue) parser.add_argument( '-s', '--stopJobs', action='store_true', help='''Stop running jobs using the value if IPP.StopJobsCommand, currently "%s". (Ignored for mode "start".)''' % defaultStopJobsCmd) parser.add_argument( '-w', '--workDir', type=str, default=defaultWorkDir, help='''Where to run the ipcluster command. Overrides the value of config var IPP.WorkDir, currently '%s'.''' % defaultWorkDir) return parser # for auto-doc generation
def createScenario(inputFile, xmlFile, trialDir): basedir = pathjoin(trialDir, 'trial-xml', 'local-xml', GROUP, BASELINE) valuesFile = pathjoin(basedir, 'mcsValues.xml') # This requires that mcsValues.xml is updated before the func is called, so # the <InputFile> specifying the "writeFunc" must come after mcsValues.xml. mcsValues = McsValues(valuesFile) value_names = { 'pop': None, # "pop" values are 2 and 5, used directly 'dds': ['A2', 'constant'], # for other params, values are 0 or 1, used as indices 'cons_dem': ['ref', 'high'], 'ccs': ['ref', 'no'], 're': ['ref', 'adv'], 'bev': ['ref', 'adv'], 'nuc_ret': ['ref', 'fast'] } def value_by_name(param): value = int( mcsValues.valueForRegion(param, 'global') ) # all values are int, though MCS system renders as floats names = value_names[param] return value if names is None else names[ value] # use trial value as index into names list # copy the trial values for these parameters into a dictionary keyed by param name v = {param: value_by_name(param) for param in value_names.keys()} pop = v['pop'] dds = v['dds'] ccs = v['ccs'] re = v['re'] bev = v['bev'] cons_dem = v['cons_dem'] nuc_ret = v['nuc_ret'] # construct a mnemonic scenario name scenario = f"SSP{pop}-CI{dds}-CD{cons_dem}-CCS{ccs}-RE{re}-BEV{bev}-NUC{nuc_ret}" _logger.info("(writeFunc) running scenario '%s'", scenario) # tree = xmlFile.tree # useful IFF tree refers to config.xml from pygcam.mcs.XMLConfigFile import XMLConfigFile, COMPONENTS_GROUP # from pygcam.xmlEditor import xmlEdit # context = 'whatever' # cfg = XMLConfigFile.getConfigForScenario(context, useCopy=False) # Or this? from pygcam.xmlSetup import scenarioEditor xmlSrcDir = getParam( 'GCAM.XmlSrc') # or is it a subdir of this that we need? refWorkspace = getParam('MCS.RunWorkspace') ed = scenarioEditor('mcs', baseline='mcs', xmlSrcDir=xmlSrcDir, refWorkspace=refWorkspace, mcsMode='trial') cfg_path = ed.cfgPath() print(f"writeFuncs.py: config path is {cfg_path}") # ed.addScenarioComponent(name, pathname) # cfg.insertComponentPathname(name, pathname, after) # cfg.updateComponentPathname(name, pathname) # cfg.deleteComponent(name) # <add name="SSP">../input/policy/scenarios/pop_gdp_{pop}.xml</add> ed.addScenarioComponent("SSP", f"../input/policy/scenarios/pop_gdp_{pop}.xml") # <replace name="dd_usa">../input/policy/scenarios/HDDCDD_{dds}.xml</replace> ed.updateScenarioComponent("dd_usa", f"../input/policy/scenarios/HDDCDD_{dds}.xml") # <add name="trn_pe_zero">../input/policy/scenarios/trn_pe_zero.xml</add> # <add name="trn_dem">../input/policy/scenarios/td_{pop}_{cons_dem}.xml</add> # <add name="FLSP">../input/policy/scenarios/FLSP_{pop}_{cons_dem}.xml</add> ed.addScenarioComponent("trn_pe_zero", "../input/policy/scenarios/trn_pe_zero.xml") ed.addScenarioComponent( "trn_dem", f"../input/policy/scenarios/td_{pop}_{cons_dem}.xml") ed.addScenarioComponent( "FLSP", f"../input/policy/scenarios/FLSP_{pop}_{cons_dem}.xml") # <if value1="{ccs}" value2="no"> # <add name="noccs_st">../input/policy/scenarios/noccs_st.xml</add> # <add name="noccs_pt">../input/policy/scenarios/noccs_pt.xml</add> # </if> if ccs == "no": ed.addScenarioComponent("noccs_st", "../input/policy/scenarios/noccs_st.xml") ed.addScenarioComponent("noccs_pt", "../input/policy/scenarios/noccs_pt.xml") # <if value1="{re}" value2="adv"> # <add name="re_adv">../input/policy/scenarios/adv_re_tech.xml</add> # </if> if re == "adv": ed.addScenarioComponent("re_adv", "../input/policy/scenarios/adv_re_tech.xml") # <if value1="{bev}" value2="adv"> # <replace name="bev_UCD">../input/policy/scenarios/bev_UCD_adv.xml</replace> # <insert name="bev_USA_adv" after="bev_USA">../input/policy/scenarios/bev_USA_adv.xml</insert> # <add name="BEV_LoBatt_compact">../input/policy/reference/BEV_LoBatt_compact_wLearning.xml</add> # <add name="BEV_LoBatt_large">../input/policy/reference/BEV_LoBatt_large_wLearning.xml</add> # <add name="BEV_LoBatt_lttruck_SUV">../input/policy/reference/BEV_LoBatt_lttruck_SUV_wLearning.xml</add> # <add name="BEV_LoBatt_midsize">../input/policy/reference/BEV_LoBatt_midsize_wLearning.xml</add> # <add name="BEV_LDV_SW">../input/policy/reference/BEV_SW_update-global.xml</add> # </if> if bev == "adv": ed.updateScenarioComponent( "bev_UCD", "../input/policy/scenarios/bev_UCD_adv.xml") ed.insertScenarioComponent( "bev_USA_adv", "../input/policy/scenarios/bev_USA_adv.xml", "bev_USA") ed.addScenarioComponent( "BEV_LoBatt_compact", "../input/policy/reference/BEV_LoBatt_compact_wLearning.xml") ed.addScenarioComponent( "BEV_LoBatt_large", "../input/policy/reference/BEV_LoBatt_large_wLearning.xml") ed.addScenarioComponent( "BEV_LoBatt_lttruck_SUV", "../input/policy/reference/BEV_LoBatt_lttruck_SUV_wLearning.xml") ed.addScenarioComponent( "BEV_LoBatt_midsize", "../input/policy/reference/BEV_LoBatt_midsize_wLearning.xml") ed.addScenarioComponent( "BEV_LDV_SW", "../input/policy/reference/BEV_SW_update-global.xml") # <if value1="{nuc_ret}" value2="fast"> # <function name="multiply">tag="nuc_usa",xpath="//half-life",value=0.6</function> # <add name="nuc_bound">../input/policy/scenarios/nonewnuclear.xml</add> # </if> if nuc_ret == "fast": # <function name="multiply">tag="nuc_usa",xpath="//half-life",value=0.6</function> ed.multiply("nuc_usa", "//half-life", 0.6) ed.addScenarioComponent("nuc_bound", "../input/policy/scenarios/nonewnuclear.xml") cfg.write(path=configPath)
def addArgs(self, parser): parser.add_argument( '--delete', action='store_true', help='''DELETE and recreate the simulation "run" directory.''') parser.add_argument( '-D', '--desc', type=str, default='', help='A brief (<= 256 char) description the simulation.') parser.add_argument( '-e', '--exportVars', default='', help= 'Export variable and distribution info in a tab-delimited file with the given name and exit.' ) parser.add_argument( '-g', '--groupName', default='', help='''The name of a scenario group to process.''') parser.add_argument( '-m', '--method', choices=['montecarlo', 'sobol', 'fast', 'morris'], default='montecarlo', help= '''Use the specified method to generate trial data. Default is "montecarlo".''' ) parser.add_argument( '-o', '--outFile', help='''For methods other than "montecarlo". The path to a "package directory" into which SALib-related data are stored. If the filename does not end in '.sa', this extension is added. The file 'problem.csv' within the package directory will contain the parameter specs in SALib format. The file inputs.csv is also generated in the file package using the chosen method's sampling method. If an outFile is not specified, a package of the name 'data.sa' is created in the simulation run-time directory.''' ) parser.add_argument( '-p', '--paramFile', default=None, help='''Specify an XML file containing parameter definitions. Defaults to the value of config parameter MCS.ParametersFile (currently %s)''' % getParam('MCS.ParametersFile')) runRoot = getParam('MCS.Root') parser.add_argument( '-r', '--runRoot', default=None, help= '''Root of the run-time directory for running user programs. Defaults to value of config parameter MCS.Root (currently %s)''' % runRoot) parser.add_argument( '-S', '--calcSecondOrder', action='store_true', help= '''For Sobol method only -- calculate second-order sensitivities.''' ) parser.add_argument('-s', '--simId', type=int, default=1, help='The id of the simulation. Default is 1.') parser.add_argument( '-t', '--trials', type=int, default=-1, help= '''The number of trials to create for this simulation (REQUIRED). If a value of 0 is given, scenario setup is performed, scenario names are added to the database, and meta-data is copied, but new trial data is not generated.''' ) return parser # for auto-doc generation
def defaultGroupName(cls): from pygcam.config import getParam projectName = getParam('GCAM.ProjectName') obj = cls.readProjectFile(projectName) return obj.scenarioSetup.defaultGroup
def genSimulation(simId, trials, paramPath, args): ''' Generate a simulation based on the given parameters. ''' from ..context import Context from ..Database import getDatabase from ..XMLParameterFile import XMLParameterFile from ..util import getSimParameterFile, getSimResultFile, symlink, filecopy from pygcam.constants import LOCAL_XML_NAME from pygcam.project import Project from pygcam.xmlSetup import ScenarioSetup runInputDir = getParam('MCS.RunInputDir') runWorkspace = getParam('MCS.RunWorkspace') # Add symlink to workspace's input dir so we can find XML files using rel paths in config files simDir = getSimDir(simId, create=True) simInputDir = os.path.join(simDir, 'input') symlink(runInputDir, simInputDir) # Ditto for workspace's local-xml workspaceLocalXml = os.path.join(runWorkspace, LOCAL_XML_NAME) simLocalXmlDir = os.path.join(simDir, LOCAL_XML_NAME) symlink(workspaceLocalXml, simLocalXmlDir) projectName = getParam('GCAM.ProjectName') project = Project.readProjectFile(projectName, groupName=args.groupName) args.groupName = groupName = args.groupName or project.scenarioSetup.defaultGroup # Run static setup for all scenarios in the given group runStaticSetup(runWorkspace, project, groupName) # TBD: Use pygcam scenario def and copy pygcam files, too scenarioFile = getParam('GCAM.ScenarioSetupFile') scenarioSetup = ScenarioSetup.parse(scenarioFile) scenarioNames = scenarioSetup.scenariosInGroup(groupName) baseline = scenarioSetup.baselineForGroup(groupName) # Copy the user's results.xml file to {simDir}/app-xml userResultFile = getParam('MCS.ResultsFile') simResultFile = getSimResultFile(simId) mkdirs(os.path.dirname(simResultFile)) filecopy(userResultFile, simResultFile) paramFileObj = XMLParameterFile(paramPath) context = Context(projectName=args.projectName, simId=simId, groupName=groupName) paramFileObj.loadInputFiles(context, scenarioNames, writeConfigFiles=True) # Define the experiments (scenarios) in the database db = getDatabase() db.addExperiments(scenarioNames, baseline, scenarioFile) if not trials: _logger.warn("Simulation meta-data has been copied.") return paramFileObj.generateRandomVars() _logger.info("Generating %d trials to %r", trials, simDir) df = genTrialData(simId, trials, paramFileObj, args) # Save generated values to the database for post-processing saveTrialData(df, simId) # Also save the param file as parameters.xml, for reference only simParamFile = getSimParameterFile(simId) filecopy(paramPath, simParamFile)