def runTrial(self): """ Run a single trial on the current engine using the local Worker. :return: (WorkerResult) holds run identification info and completion status """ context = self.context runDir = context.getScenarioDir(create=True) _logger.info("runDir is %s", runDir) os.chdir(runDir) trialDir = os.path.dirname(runDir) logDir = os.path.join(trialDir, 'log') mkdirs(logDir) if not self.runLocal: logFile = os.path.join(logDir, context.scenario + '.log') setParam('GCAM.LogFile', logFile) setParam('GCAM.LogConsole', 'False') # avoids duplicate output to file configureLogs(force=True) self.setStatus(RUN_RUNNING) result = self._runTrial() return result
def _dirFromNumber(n, prefix="", create=False): ''' Compute a directory name using a 2-level directory structure that allows 1000 nodes at each level, accommodating up to 1 million files (0 to 999,999) in two levels. ''' from numpy import log10 # lazy import maxnodes = getParamAsInt('MCS.MaxSimDirs') or 1000 # Require a power of 10 log = log10(maxnodes) if log != int(log): raise PygcamMcsUserError( "MaxSimDirs must be a power of 10 (default value is 1000)") log = int(log) level1 = n // maxnodes level2 = n % maxnodes directory = os.path.join(prefix, str(level1).zfill(log), str(level2).zfill(log)) if create: mkdirs(directory) return directory
def makePlotPath(value, simId): plotDir = getParam('MCS.PlotDir') subDir = os.path.join(plotDir, "s%d" % simId) mkdirs(subDir) plotType = getParam('MCS.PlotType') path = os.path.join(subDir, "%s.%s" % (value, plotType)) #print "Plot path: ", path return path
def setUp(self): self.ws = './data/ws' self.baseline = 'base-0' self.policy = 'corn-0' self.queryDir = 'queryResults' self.years = [2015, 2050] self.tmpDir = '/tmp/testDiff' self.removeTmpDir() # start clean mkdirs(self.tmpDir)
def getScenarioDir(self, create=False): ''' Return and optionally create the path to the directory for a given experiment. ''' trialDir = self.getTrialDir(create=False) scenarioDir = os.path.join(trialDir, self.scenario) if create: mkdirs(scenarioDir) return scenarioDir
def copyfiles(files, dstdir, removeDst=True): ''' :param files: a list of files to copy :param dstdir: the directory to copy to :param removeDst: if True-like, remove destination file before copying :return: nothing ''' mkdirs(dstdir) for f in files: filecopy(f, dstdir, removeDst=removeDst)
def createOutputDir(outputDir): from ..utils import removeFileOrTree from ..temp_file import getTempDir removeFileOrTree(outputDir, raiseError=False) tempOutputDir = getParam('MCS.TempOutputDir') if tempOutputDir: # We create this on /scratch which is purged automatically. newDir = getTempDir(suffix='', tmpDir=tempOutputDir, delete=False) mkdirs(newDir) _logger.debug("Creating '%s' link to %s" % (outputDir, newDir)) symlink(newDir, outputDir) else: mkdirs(outputDir)
def getSimDir(simId, create=False): ''' Return and optionally create the path to the top-level simulation directory for the given simulation number, based on the SimsDir parameter specified in the config file. ''' simsDir = getParam('MCS.RunSimsDir') if not simsDir: raise PygcamMcsUserError( "Missing required config parameter 'RunSimsDir'") simDir = os.path.join(simsDir, 's%03d' % simId) # name is of format ".../s001/" if create: mkdirs(simDir) return simDir
def runStaticSetup(runWorkspace, project, groupName): """ Run the --staticOnly setup in the MCS copy of the workspace, for all scenarios. This is called from gensim, so we fake the "context" for trial 0, since all trials have local-xml symlinked to RunWorkspace's local-xml. """ import pygcam.tool from pygcam.utils import mkdirs from pygcam.constants import LOCAL_XML_NAME from ..util import symlink from ..error import GcamToolError projectName = project.projectName scenarios = project.getKnownScenarios() scenariosArg = ','.join(scenarios) useGroupDir = project.scenarioGroup.useGroupDir groupSubdir = groupName if useGroupDir else '' # create symlinks from all the scenarios' local-xml dirs to shared one # under {projectName}/Workspace sandboxDir = os.path.join(runWorkspace, groupSubdir) mkdirs(sandboxDir) wsXmlDir = os.path.join(runWorkspace, LOCAL_XML_NAME) mkdirs(wsXmlDir) for scenario in scenarios: dirname = os.path.join(sandboxDir, scenario) mkdirs(dirname) linkname = os.path.join(dirname, LOCAL_XML_NAME) symlink(wsXmlDir, linkname) # N.B. RunWorkspace for gensim is pygcam's RefWorkspace toolArgs = [ '+P', projectName, '--mcs=gensim', 'run', '-s', 'setup', '-S', scenariosArg, '--sandboxDir=' + sandboxDir ] if useGroupDir: toolArgs += ['-g', groupName] _logger.debug('Running: %s', 'gt ' + ' '.join(toolArgs)) status = pygcam.tool.main(argv=toolArgs, raiseError=True) if status != 0: msg = '"gt setup" exited with status %d' % status raise GcamToolError(msg) return status
def genSimulation(simId, trials, paramPath, args): ''' Generate a simulation based on the given parameters. ''' from ..context import Context from ..Database import getDatabase from ..XMLParameterFile import XMLParameterFile from ..util import getSimParameterFile, getSimResultFile, symlink, filecopy from pygcam.constants import LOCAL_XML_NAME from pygcam.project import Project from pygcam.xmlSetup import ScenarioSetup runInputDir = getParam('MCS.RunInputDir') runWorkspace = getParam('MCS.RunWorkspace') # Add symlink to workspace's input dir so we can find XML files using rel paths in config files simDir = getSimDir(simId, create=True) simInputDir = os.path.join(simDir, 'input') symlink(runInputDir, simInputDir) # Ditto for workspace's local-xml workspaceLocalXml = os.path.join(runWorkspace, LOCAL_XML_NAME) simLocalXmlDir = os.path.join(simDir, LOCAL_XML_NAME) symlink(workspaceLocalXml, simLocalXmlDir) projectName = getParam('GCAM.ProjectName') project = Project.readProjectFile(projectName, groupName=args.groupName) args.groupName = groupName = args.groupName or project.scenarioSetup.defaultGroup # Run static setup for all scenarios in the given group runStaticSetup(runWorkspace, project, groupName) # TBD: Use pygcam scenario def and copy pygcam files, too scenarioFile = getParam('GCAM.ScenarioSetupFile') scenarioSetup = ScenarioSetup.parse(scenarioFile) scenarioNames = scenarioSetup.scenariosInGroup(groupName) baseline = scenarioSetup.baselineForGroup(groupName) # Copy the user's results.xml file to {simDir}/app-xml userResultFile = getParam('MCS.ResultsFile') simResultFile = getSimResultFile(simId) mkdirs(os.path.dirname(simResultFile)) filecopy(userResultFile, simResultFile) paramFileObj = XMLParameterFile(paramPath) context = Context(projectName=args.projectName, simId=simId, groupName=groupName) paramFileObj.loadInputFiles(context, scenarioNames, writeConfigFiles=True) # Define the experiments (scenarios) in the database db = getDatabase() db.addExperiments(scenarioNames, baseline, scenarioFile) if not trials: _logger.warn("Simulation meta-data has been copied.") return paramFileObj.generateRandomVars() _logger.info("Generating %d trials to %r", trials, simDir) df = genTrialData(simId, trials, paramFileObj, args) # Save generated values to the database for post-processing saveTrialData(df, simId) # Also save the param file as parameters.xml, for reference only simParamFile = getSimParameterFile(simId) filecopy(paramPath, simParamFile)
def genSALibData(trials, method, paramFileObj, args): from ..error import PygcamMcsUserError from ..sensitivity import DFLT_PROBLEM_FILE, Sobol, FAST, Morris # , MonteCarlo from pygcam.utils import ensureExtension, removeTreeSafely, mkdirs SupportedDistros = ['Uniform', 'LogUniform', 'Triangle', 'Linked'] outFile = args.outFile or os.path.join(getSimDir(args.simId), 'data.sa') outFile = ensureExtension(outFile, '.sa') if os.path.lexists(outFile): # Attempt to mv an existing version of the file to the same name with '~' backupFile = outFile + '~' if os.path.isdir(backupFile): removeTreeSafely(backupFile) elif os.path.lexists(backupFile): raise PygcamMcsUserError( "Refusing to delete '%s' since it's not a file package." % backupFile) os.rename(outFile, backupFile) mkdirs(outFile) linked = [] problemFile = pathjoin(outFile, DFLT_PROBLEM_FILE) with open(problemFile, 'w') as f: f.write('name,low,high\n') for elt in paramFileObj.tree.iterfind('//Parameter'): name = elt.get('name') distElt = elt.find('Distribution') distSpec = distElt[0] # should have only one child as per schema distName = distSpec.tag # These 3 distro forms specify min and max values, which we use with SALib legal = SupportedDistros + ['Linked'] if distName not in legal: raise PygcamMcsUserError( "Found '%s' distribution; must be one of %s for use with SALib." % (distName, legal)) if distName == 'Linked': # TBD: ignore these and figure it out when loading the file? linked.append((name, distSpec.get('parameter'))) continue # Parse out the various forms: (max, min), (factor), (range), and factor for LogUniform attrib = distSpec.attrib if 'min' in attrib and 'max' in attrib: minValue = float(attrib['min']) maxValue = float(attrib['max']) elif 'factor' in attrib: value = float(attrib['factor']) if distName == 'LogUniform': minValue = 1 / value maxValue = value else: minValue = 1 - value maxValue = 1 + value elif 'range' in attrib: value = float(attrib['range']) minValue = -value maxValue = value f.write("%s,%f,%f\n" % (name, minValue, maxValue)) methods = (Sobol, FAST, Morris) # , MonteCarlo) methodMap = {cls.__name__.lower(): cls for cls in methods} cls = methodMap[method] sa = cls(outFile) # saves to input.csv in file package sa.sample(trials=trials, calc_second_order=args.calcSecondOrder) return sa.inputsDF