def __makePBS(self): """ Creates the run.pbs file used by Hopper's batch system """ #Calculate number of mpi nodes needed, and build the commands jobs = self.__paramFile.dataDict["jobs"] commands = [] numNodes = 0 for jobName, job in jobs: command = None if DataDict.boolconv(job["aprun"]): if job["tasksNode"] == "-1": nodesThis = int(math.ceil(int(job["tasks"])/float(self.CPUsPerNode))) else: assert int(job["tasksNode"]) <= self.CPUsPerNode nodesThis = int(math.ceil(int(job["tasks"])/float(job["tasksNode"]))) if nodesThis > numNodes: numNodes = nodesThis def makeOption(optionName, key, optional): get = job[key] if get == "-1" and optional: return "" return optionName + " " + get + " " command = "aprun " + makeOption("-n", "tasks", False)\ + makeOption("-N", "tasksNode", True)\ + makeOption("-S", "tasksNuma", True)\ + job["command"] + " " + job["commandArgs"] else: command = job["command"] commands.append(command) if len(commands) == 0: raise AcdOptiException_optiRunner_stageError("No commands built") if not numNodes > 0: raise AcdOptiException_optiRunner_stageError("Got numNodes="+str(numNodes)) #Write PBS file header runpbs = open(os.path.join(self.runConfig.stageFolder, "run.pbs"), 'w') runpbs.write("#!/bin/bash\n") torqueMeta = self.getTorqueMeta() runpbs.write("#PBS -q " + torqueMeta["queue"] + "\n") runpbs.write("#PBS -l mppwidth=" + str(numNodes*self.CPUsPerNode) + "\n") runpbs.write("#PBS -l walltime=" + torqueMeta["walltime"] + "\n") runpbs.write("#PBS -N " + self.runConfig.stageName + "\n") runpbs.write("#PBS -A " + torqueMeta["repo"] + "\n") if DataDict.boolconv(torqueMeta["importVars"]): runpbs.write("#PBS -V\n") runpbs.write("\n\n") #Write PBS script runpbs.write("## Commands:\n") for command in commands: runpbs.write(command + "\n") runpbs.close()
def __genMetaOptions(childDict): """ Recursive method used by createNew() for generating the "options" part of the metaFile. Returns a DataDict which becomes one branch/level of the metaFile """ print "AcdOptiSolverSetupManager::__genMetaOptions()" ret = DataDict() for item in childDict: # Make a copy of the old items in the childDict and add it to the metFile thisItem = ret.pushBack(item[0], item[1].copy()) # Add some new meta-information fields if thisItem.getValSingle("type") == "dict": # Recursively modify this dicts children also thisItem.setValSingle( "children", AcdOptiSolverManager.__genMetaOptions(item[1].getValSingle("children")) ) else: # Ordinary data field, set its value to whatever is default thisItem.pushBack("value", thisItem.getValSingle("default")) if DataDict.boolconv(item[1].getValSingle("must")): thisItem.pushBack("enabled", "True") else: thisItem.pushBack("enabled", "False") return ret
def __init__(self,folder,collection): self.folder = folder self.collection = collection #Construct the instance name from folder instName = folder if instName[-1] == "/": instName = instName[0:-1] self.instName = instName = os.path.split(instName)[1] #Load paramFile self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw') if self.__paramfile.dataDict["fileID"] != "AcdOptiMetaAnalysis": raise AcdOptiException_metaAnalysis_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'") if self.__paramfile.dataDict.getValSingle("instName") != self.instName: raise AcdOptiException_metaAnalysis_loadFail("instName doesn't match folder name") self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown")) self.xVariable = self.__paramfile.dataDict["xVariable"] self.yVariable = self.__paramfile.dataDict["yVariable"] if len(self.__paramfile.dataDict.getVals("fVariable")) == 0: self.__paramfile.dataDict.pushBack("fVariable", "") self.__paramfile.dataDict.pushBack("fEquals", "") self.__paramfile.dataDict.pushBack("fLT", "") self.__paramfile.dataDict.pushBack("fGT", "") self.__paramfile.write() self.fVariable = self.__paramfile.dataDict["fVariable"] def floatOrNone(strIn): if strIn == "": return None else: return float(strIn) self.fEquals = floatOrNone(self.__paramfile.dataDict["fEquals"]) self.fGT = floatOrNone(self.__paramfile.dataDict["fGT"]) self.fLT = floatOrNone(self.__paramfile.dataDict["fLT"]) if len(self.__paramfile.dataDict.getVals("targetValue")) > 0: self.targetValue = float(self.__paramfile.dataDict["targetValue"]) else: self.targetValue = None anaData = self.__paramfile.dataDict["anaData"] self.xArray = [] self.yArray = [] for (x,y) in anaData: self.xArray.append(float(x)) self.yArray.append(float(y)) #Final init & lockdown checks if self.lockdown == True: if self.yVariable == None or self.xVariable == None: #The x,yArray may still be empty, don't check for this raise AcdOptiException_metaAnalysis_loadFail("Lockdown, but xVariable='" + self.xVariable + "', yVariable='" + self.yVariable) self.xArray = np.asarray(self.xArray) self.yArray = np.asarray(self.yArray)
def __generateSetup_recursiveHelper(setupDict): print "AcdOptiSolverManager::__generateSetup_recursiveHelper()" # , setupDict=", setupDict ret = DataDict() for item in setupDict: if not DataDict.boolconv(item[1]["enabled"]): continue if item[1]["type"] == "dict": ret.pushBack(item[0], AcdOptiSolverManager.__generateSetup_recursiveHelper(item[1]["children"])) else: ret.pushBack(item[0], item[1]["value"]) return ret
def __init__(self, folder, collection): self.folder = folder self.collection = collection #Construct the instance name from folder instName = folder if instName[-1] == "/": instName = instName[0:-1] self.instName = instName = os.path.split(instName)[1] #Load paramFile self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw') if self.__paramfile.dataDict["fileID"] != "AcdOptiDataExtractor": raise AcdOptiException_dataExtractor_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'") if self.__paramfile.dataDict.getValSingle("instName") != self.instName: raise AcdOptiException_dataExtractor_loadFail("instName doesn't match folder name") self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown")) try: self.extractFname = self.__paramfile.dataDict["extractFname"] except AcdOptiException_dataDict_getValsSingle: print "AcdOptiDataExtractor::__init__(): Adding extractFname to paramFile" self.__paramfile.dataDict.pushBack("extractFname", "") self.__paramfile.write() self.extractFname = "" #KeepKeys self.keepKeys = [] try: fileKeepKeys = self.__paramfile.dataDict["keepKeys"] for (k,v) in fileKeepKeys: assert k == "key" self.keepKeys.append(v) except AcdOptiException_dataDict_getValsSingle: print "AcdOptiDataExtractor::__init__(): Adding keepKeys to paramFile" self.__paramfile.dataDict.pushBack("keepKeys", DataDict()) self.__paramfile.write() #Parse keyNames self.keyNames = [] for (k,v) in self.__paramfile.dataDict["keyNames"]: assert k == "keyName" self.keyNames.append(v) #Parse extractedData self.dataExtracted = [] for (k,v) in self.__paramfile.dataDict["extractedData"]: assert k == "dataPoint" assert isinstance(v,DataDict) pb = {} for (k,v) in v: pb[k] = v if not k in self.keyNames: raise AcdOptiException_dataExtractor_loadFail("Key name '" + k + "' not found in self.keyNames") self.dataExtracted.append(pb) #Find, load, and attach filters self.filters = [] for (k,v) in self.__paramfile.dataDict["filters"]: self.filters.append(AcdOptiDataExtractorFilter.getFilterClass(k,v)) #Plots self.plots = [] try: filePlots = self.__paramfile.dataDict["plots"] for (k,v) in filePlots: self.plots.append(AcdOptiDataExtractorPlot.getPlotClass(k, self, v)) except AcdOptiException_dataDict_getValsSingle: self.__paramfile.dataDict.pushBack("plots", DataDict()) self.__paramfile.write()
def __init__(self, folder, scanCollection): self.folder = folder self.scanCollection = scanCollection #Construct the instance name from folder instName = folder if instName[-1] == "/": instName = instName[0:-1] self.instName = instName = os.path.split(instName)[1] #Load paramFile self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw') if self.__paramfile.dataDict["fileID"] != "AcdOptiScan": raise AcdOptiException_scan_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'") if self.__paramfile.dataDict.getValSingle("instName") != self.instName: raise AcdOptiException_scan_loadFail("instName doesn't match folder name") self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown")) self.staged = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("staged")) self.run = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("run")) self.slaveGeomsDict = self.__paramfile.dataDict["slaveGeoms"] self.slaveGeoms = [] baseGeomInstance_name = self.__paramfile.dataDict["baseGeomInstance_name"] if baseGeomInstance_name == "": self.baseGeomInstance = None assert len(self.slaveGeomsDict) == 0 else: self.baseGeomInstance = self.scanCollection.project.geomCollection.geomInstances[baseGeomInstance_name] if self.__paramfile.dataDict["scanParameter_name"] != "": #assert self.baseGeomInstance != None #Why? self.scanParameter_name = self.__paramfile.dataDict["scanParameter_name"] assert self.scanParameter_name in self.getValidParamNames() self.scanParameter_range_max = float(self.__paramfile.dataDict["scanParameter_max"]) self.scanParameter_range_min = float(self.__paramfile.dataDict["scanParameter_min"]) self.scanParameter_range_step = float(self.__paramfile.dataDict["scanParameter_step"]) self.generateRange() try: self.predict_anaVariable = self.__paramfile.dataDict["predict_anaVariable"] self.predict_targetValue = self.__paramfile.dataDict["predict_targetValue"] except AcdOptiException_dataDict_getValsSingle: self.predict_anaVariable = "" self.predict_targetValue = "" self.__paramfile.dataDict.pushBack("predict_anaVariable", "") self.__paramfile.dataDict.pushBack("predict_targetValue", "") self.__paramfile.write() try: self.predict_a = self.__paramfile.dataDict["predict_a"] self.predict_b = self.__paramfile.dataDict["predict_b"] self.predict_x = self.__paramfile.dataDict["predict_x"] self.predict_r = self.__paramfile.dataDict["predict_r"] except AcdOptiException_dataDict_getValsSingle: self.predict_a = "" self.predict_b = "" self.predict_x = "" self.predict_r = "" self.__paramfile.dataDict.pushBack("predict_a", "") self.__paramfile.dataDict.pushBack("predict_b", "") self.__paramfile.dataDict.pushBack("predict_x", "") self.__paramfile.dataDict.pushBack("predict_r", "") self.__paramfile.write() try: self.predict_ndof=self.__paramfile.dataDict["predict_ndof"] except AcdOptiException_dataDict_getValsSingle: self.predict_ndof = "" self.__paramfile.dataDict.pushBack("predict_ndof","") self.__paramfile.write() for (geomName, nothingOfInterest) in self.slaveGeomsDict: #Mutal referencing self.slaveGeoms.append(self.scanCollection.project.geomCollection.geomInstances[geomName]) self.slaveGeoms[-1].scanInstances.append(self)
def __init__(self, folder, geometryInstance, meshTemplateCollection): """ Loads the meshInstance from the data from folder and connects it to its template and geometryInstance. Raises AcdOptiException_meshInstance_loadFail if something (such as "not a meshInstance folder") goes wrong. """ print "AcdOptiMeshInstance::__init__()" self.folder = folder self.geometryInstance = geometryInstance self.meshTemplateCollection = meshTemplateCollection if not os.path.isdir(folder): raise AcdOptiException_meshInstance_loadFail(\ "Folder \"" + folder + "\" does not exist") #Construct the instance name from folder name instname = folder if instname[-1] == "/": instname = instname[0:-1] instname = os.path.split(instname)[1] self.instName = instname #Load paramFile.set try: self.__paramFile = AcdOptiFileParser_simple(\ os.path.join(folder, "paramFile.set"), 'rw') except IOError: raise AcdOptiException_meshInstance_loadFail(\ "Problem loading file \"paramFile.set\" in folder \"" +\ folder + "\", got an IOError") if self.__paramFile.dataDict.getValSingle("fileID") != "meshInstance": raise AcdOptiException_meshInstance_loadFail( "Wrong fileID in meshInstance.set, got \"" + self.__paramFile.dataDict.getValSingle("fileID") + "\"") lock = self.__paramFile.dataDict.getValSingle("lockdown") if lock == "True": self.lockdown = True elif lock == "False": self.lockdown = False else: raise AcdOptiException_meshInstance_loadFail(\ "Invalid content in field \"lockdown\" of file paramFile.set") meshBad = self.__paramFile.dataDict.getVals("meshBad") if len(meshBad) == 0: self.meshBad = False else: assert len(meshBad) == 1 self.meshBad = DataDict.boolconv(meshBad[0]) #Check that the geometryInstance is correct geometryInstance_expectedName = self.__paramFile.dataDict.getValSingle("geomInstance_name") if geometryInstance.instName != geometryInstance_expectedName: raise AcdOptiException_meshInstance_loadFail(\ "Excpected name of geometryInstance does not match the one passed") #Find the MeshTemplate meshTemplate_expectedName = self.__paramFile.dataDict.getValSingle("meshTemplate_name") try: self.meshTemplate = self.meshTemplateCollection.meshTemplates[meshTemplate_expectedName] self.meshTemplate.registerInstance(self) except KeyError: raise AcdOptiException_meshInstance_loadFail("Could not find the meshTemplate \"" + meshTemplate_expectedName + "\"") #Find and load template overrides self.__templateOverrides = {} try: templateOverrides_data = self.__paramFile.dataDict.getValSingle("templateOverrides") except AcdOptiException_dataDict_getValsSingle: raise AcdOptiException_meshInstance_loadFail\ ("Couldn't load templateOverrides from paramFile.set") if not isinstance(templateOverrides_data,DataDict): raise AcdOptiException_meshInstance_loadFail\ ("templateOverrides from paramFile.set is not a DataDict!") for (k,v) in zip(templateOverrides_data.keys, templateOverrides_data.vals): if k in self.__templateOverrides: raise AcdOptiException_geomCollection_loadFail\ ("Double occurrence of key \"" + k + "\" in templateOverrides on file") if not k in self.meshTemplate.paramDefaults_getKeys(): raise AcdOptiException_meshInstance_loadFail( "Entry \"" + k + "\" in templateOverrides on file has no match in the template") self.__templateOverrides[k] = v #Find subfolders and check if they are runConfigs if not os.path.isdir(os.path.join(self.folder, "runConfigs")): raise AcdOptiException_meshInstance_loadFail("Could not find runConfigs folder") self.runConfigs = {} for d in os.listdir(os.path.join(self.folder,"runConfigs")): dAbs = os.path.abspath(os.path.join(self.folder,"runConfigs",d)) if not os.path.isdir(dAbs): #Skip files etc. continue #try: self.runConfigs[d] = AcdOptiRunConfig(dAbs,self)