def runTrial(self):
"""
Run a single trial on the current engine using the local Worker.
:return: (WorkerResult) holds run identification info and completion status
"""
context = self.context
runDir = context.getScenarioDir(create=True)
_logger.info("runDir is %s", runDir)
os.chdir(runDir)
trialDir = os.path.dirname(runDir)
logDir = os.path.join(trialDir, 'log')
mkdirs(logDir)
if not self.runLocal:
logFile = os.path.join(logDir, context.scenario + '.log')
setParam('GCAM.LogFile', logFile)
setParam('GCAM.LogConsole',
'False') # avoids duplicate output to file
configureLogs(force=True)
self.setStatus(RUN_RUNNING)
result = self._runTrial()
return result
def _dirFromNumber(n, prefix="", create=False):
'''
Compute a directory name using a 2-level directory structure that
allows 1000 nodes at each level, accommodating up to 1 million files
(0 to 999,999) in two levels.
'''
from numpy import log10 # lazy import
maxnodes = getParamAsInt('MCS.MaxSimDirs') or 1000
# Require a power of 10
log = log10(maxnodes)
if log != int(log):
raise PygcamMcsUserError(
"MaxSimDirs must be a power of 10 (default value is 1000)")
log = int(log)
level1 = n // maxnodes
level2 = n % maxnodes
directory = os.path.join(prefix,
str(level1).zfill(log),
str(level2).zfill(log))
if create:
mkdirs(directory)
return directory
def makePlotPath(value, simId):
plotDir = getParam('MCS.PlotDir')
subDir = os.path.join(plotDir, "s%d" % simId)
mkdirs(subDir)
plotType = getParam('MCS.PlotType')
path = os.path.join(subDir, "%s.%s" % (value, plotType))
#print "Plot path: ", path
return path
def setUp(self):
self.ws = './data/ws'
self.baseline = 'base-0'
self.policy = 'corn-0'
self.queryDir = 'queryResults'
self.years = [2015, 2050]
self.tmpDir = '/tmp/testDiff'
self.removeTmpDir() # start clean
mkdirs(self.tmpDir)
def getScenarioDir(self, create=False):
'''
Return and optionally create the path to the directory for a given experiment.
'''
trialDir = self.getTrialDir(create=False)
scenarioDir = os.path.join(trialDir, self.scenario)
if create:
mkdirs(scenarioDir)
return scenarioDir
def copyfiles(files, dstdir, removeDst=True):
'''
:param files: a list of files to copy
:param dstdir: the directory to copy to
:param removeDst: if True-like, remove destination file before copying
:return: nothing
'''
mkdirs(dstdir)
for f in files:
filecopy(f, dstdir, removeDst=removeDst)
def createOutputDir(outputDir):
from ..utils import removeFileOrTree
from ..temp_file import getTempDir
removeFileOrTree(outputDir, raiseError=False)
tempOutputDir = getParam('MCS.TempOutputDir')
if tempOutputDir:
# We create this on /scratch which is purged automatically.
newDir = getTempDir(suffix='', tmpDir=tempOutputDir, delete=False)
mkdirs(newDir)
_logger.debug("Creating '%s' link to %s" % (outputDir, newDir))
symlink(newDir, outputDir)
else:
mkdirs(outputDir)
def getSimDir(simId, create=False):
'''
Return and optionally create the path to the top-level simulation
directory for the given simulation number, based on the SimsDir
parameter specified in the config file.
'''
simsDir = getParam('MCS.RunSimsDir')
if not simsDir:
raise PygcamMcsUserError(
"Missing required config parameter 'RunSimsDir'")
simDir = os.path.join(simsDir,
's%03d' % simId) # name is of format ".../s001/"
if create:
mkdirs(simDir)
return simDir
def runStaticSetup(runWorkspace, project, groupName):
"""
Run the --staticOnly setup in the MCS copy of the workspace, for all scenarios.
This is called from gensim, so we fake the "context" for trial 0, since all
trials have local-xml symlinked to RunWorkspace's local-xml.
"""
import pygcam.tool
from pygcam.utils import mkdirs
from pygcam.constants import LOCAL_XML_NAME
from ..util import symlink
from ..error import GcamToolError
projectName = project.projectName
scenarios = project.getKnownScenarios()
scenariosArg = ','.join(scenarios)
useGroupDir = project.scenarioGroup.useGroupDir
groupSubdir = groupName if useGroupDir else ''
# create symlinks from all the scenarios' local-xml dirs to shared one
# under {projectName}/Workspace
sandboxDir = os.path.join(runWorkspace, groupSubdir)
mkdirs(sandboxDir)
wsXmlDir = os.path.join(runWorkspace, LOCAL_XML_NAME)
mkdirs(wsXmlDir)
for scenario in scenarios:
dirname = os.path.join(sandboxDir, scenario)
mkdirs(dirname)
linkname = os.path.join(dirname, LOCAL_XML_NAME)
symlink(wsXmlDir, linkname)
# N.B. RunWorkspace for gensim is pygcam's RefWorkspace
toolArgs = [
'+P', projectName, '--mcs=gensim', 'run', '-s', 'setup', '-S',
scenariosArg, '--sandboxDir=' + sandboxDir
]
if useGroupDir:
toolArgs += ['-g', groupName]
_logger.debug('Running: %s', 'gt ' + ' '.join(toolArgs))
status = pygcam.tool.main(argv=toolArgs, raiseError=True)
if status != 0:
msg = '"gt setup" exited with status %d' % status
raise GcamToolError(msg)
return status
def genSimulation(simId, trials, paramPath, args):
'''
Generate a simulation based on the given parameters.
'''
from ..context import Context
from ..Database import getDatabase
from ..XMLParameterFile import XMLParameterFile
from ..util import getSimParameterFile, getSimResultFile, symlink, filecopy
from pygcam.constants import LOCAL_XML_NAME
from pygcam.project import Project
from pygcam.xmlSetup import ScenarioSetup
runInputDir = getParam('MCS.RunInputDir')
runWorkspace = getParam('MCS.RunWorkspace')
# Add symlink to workspace's input dir so we can find XML files using rel paths in config files
simDir = getSimDir(simId, create=True)
simInputDir = os.path.join(simDir, 'input')
symlink(runInputDir, simInputDir)
# Ditto for workspace's local-xml
workspaceLocalXml = os.path.join(runWorkspace, LOCAL_XML_NAME)
simLocalXmlDir = os.path.join(simDir, LOCAL_XML_NAME)
symlink(workspaceLocalXml, simLocalXmlDir)
projectName = getParam('GCAM.ProjectName')
project = Project.readProjectFile(projectName, groupName=args.groupName)
args.groupName = groupName = args.groupName or project.scenarioSetup.defaultGroup
# Run static setup for all scenarios in the given group
runStaticSetup(runWorkspace, project, groupName)
# TBD: Use pygcam scenario def and copy pygcam files, too
scenarioFile = getParam('GCAM.ScenarioSetupFile')
scenarioSetup = ScenarioSetup.parse(scenarioFile)
scenarioNames = scenarioSetup.scenariosInGroup(groupName)
baseline = scenarioSetup.baselineForGroup(groupName)
# Copy the user's results.xml file to {simDir}/app-xml
userResultFile = getParam('MCS.ResultsFile')
simResultFile = getSimResultFile(simId)
mkdirs(os.path.dirname(simResultFile))
filecopy(userResultFile, simResultFile)
paramFileObj = XMLParameterFile(paramPath)
context = Context(projectName=args.projectName,
simId=simId,
groupName=groupName)
paramFileObj.loadInputFiles(context, scenarioNames, writeConfigFiles=True)
# Define the experiments (scenarios) in the database
db = getDatabase()
db.addExperiments(scenarioNames, baseline, scenarioFile)
if not trials:
_logger.warn("Simulation meta-data has been copied.")
return
paramFileObj.generateRandomVars()
_logger.info("Generating %d trials to %r", trials, simDir)
df = genTrialData(simId, trials, paramFileObj, args)
# Save generated values to the database for post-processing
saveTrialData(df, simId)
# Also save the param file as parameters.xml, for reference only
simParamFile = getSimParameterFile(simId)
filecopy(paramPath, simParamFile)
def genSALibData(trials, method, paramFileObj, args):
from ..error import PygcamMcsUserError
from ..sensitivity import DFLT_PROBLEM_FILE, Sobol, FAST, Morris # , MonteCarlo
from pygcam.utils import ensureExtension, removeTreeSafely, mkdirs
SupportedDistros = ['Uniform', 'LogUniform', 'Triangle', 'Linked']
outFile = args.outFile or os.path.join(getSimDir(args.simId), 'data.sa')
outFile = ensureExtension(outFile, '.sa')
if os.path.lexists(outFile):
# Attempt to mv an existing version of the file to the same name with '~'
backupFile = outFile + '~'
if os.path.isdir(backupFile):
removeTreeSafely(backupFile)
elif os.path.lexists(backupFile):
raise PygcamMcsUserError(
"Refusing to delete '%s' since it's not a file package." %
backupFile)
os.rename(outFile, backupFile)
mkdirs(outFile)
linked = []
problemFile = pathjoin(outFile, DFLT_PROBLEM_FILE)
with open(problemFile, 'w') as f:
f.write('name,low,high\n')
for elt in paramFileObj.tree.iterfind('//Parameter'):
name = elt.get('name')
distElt = elt.find('Distribution')
distSpec = distElt[0] # should have only one child as per schema
distName = distSpec.tag
# These 3 distro forms specify min and max values, which we use with SALib
legal = SupportedDistros + ['Linked']
if distName not in legal:
raise PygcamMcsUserError(
"Found '%s' distribution; must be one of %s for use with SALib."
% (distName, legal))
if distName == 'Linked': # TBD: ignore these and figure it out when loading the file?
linked.append((name, distSpec.get('parameter')))
continue
# Parse out the various forms: (max, min), (factor), (range), and factor for LogUniform
attrib = distSpec.attrib
if 'min' in attrib and 'max' in attrib:
minValue = float(attrib['min'])
maxValue = float(attrib['max'])
elif 'factor' in attrib:
value = float(attrib['factor'])
if distName == 'LogUniform':
minValue = 1 / value
maxValue = value
else:
minValue = 1 - value
maxValue = 1 + value
elif 'range' in attrib:
value = float(attrib['range'])
minValue = -value
maxValue = value
f.write("%s,%f,%f\n" % (name, minValue, maxValue))
methods = (Sobol, FAST, Morris) # , MonteCarlo)
methodMap = {cls.__name__.lower(): cls for cls in methods}
cls = methodMap[method]
sa = cls(outFile)
# saves to input.csv in file package
sa.sample(trials=trials, calc_second_order=args.calcSecondOrder)
return sa.inputsDF
