def __makePBS(self):
"""
Creates the run.pbs file used by Hopper's batch system
"""
#Calculate number of mpi nodes needed, and build the commands
jobs = self.__paramFile.dataDict["jobs"]
commands = []
numNodes = 0
for jobName, job in jobs:
command = None
if DataDict.boolconv(job["aprun"]):
if job["tasksNode"] == "-1":
nodesThis = int(math.ceil(int(job["tasks"])/float(self.CPUsPerNode)))
else:
assert int(job["tasksNode"]) <= self.CPUsPerNode
nodesThis = int(math.ceil(int(job["tasks"])/float(job["tasksNode"])))
if nodesThis > numNodes:
numNodes = nodesThis
def makeOption(optionName, key, optional):
get = job[key]
if get == "-1" and optional:
return ""
return optionName + " " + get + " "
command = "aprun " + makeOption("-n", "tasks", False)\
+ makeOption("-N", "tasksNode", True)\
+ makeOption("-S", "tasksNuma", True)\
+ job["command"] + " " + job["commandArgs"]
else:
command = job["command"]
commands.append(command)
if len(commands) == 0:
raise AcdOptiException_optiRunner_stageError("No commands built")
if not numNodes > 0:
raise AcdOptiException_optiRunner_stageError("Got numNodes="+str(numNodes))
#Write PBS file header
runpbs = open(os.path.join(self.runConfig.stageFolder, "run.pbs"), 'w')
runpbs.write("#!/bin/bash\n")
torqueMeta = self.getTorqueMeta()
runpbs.write("#PBS -q " + torqueMeta["queue"] + "\n")
runpbs.write("#PBS -l mppwidth=" + str(numNodes*self.CPUsPerNode) + "\n")
runpbs.write("#PBS -l walltime=" + torqueMeta["walltime"] + "\n")
runpbs.write("#PBS -N " + self.runConfig.stageName + "\n")
runpbs.write("#PBS -A " + torqueMeta["repo"] + "\n")
if DataDict.boolconv(torqueMeta["importVars"]):
runpbs.write("#PBS -V\n")
runpbs.write("\n\n")
#Write PBS script
runpbs.write("## Commands:\n")
for command in commands:
runpbs.write(command + "\n")
runpbs.close()
def __genMetaOptions(childDict):
"""
Recursive method used by createNew() for generating the "options" part of the metaFile.
Returns a DataDict which becomes one branch/level of the metaFile
"""
print "AcdOptiSolverSetupManager::__genMetaOptions()"
ret = DataDict()
for item in childDict:
# Make a copy of the old items in the childDict and add it to the metFile
thisItem = ret.pushBack(item[0], item[1].copy())
# Add some new meta-information fields
if thisItem.getValSingle("type") == "dict":
# Recursively modify this dicts children also
thisItem.setValSingle(
"children", AcdOptiSolverManager.__genMetaOptions(item[1].getValSingle("children"))
)
else:
# Ordinary data field, set its value to whatever is default
thisItem.pushBack("value", thisItem.getValSingle("default"))
if DataDict.boolconv(item[1].getValSingle("must")):
thisItem.pushBack("enabled", "True")
else:
thisItem.pushBack("enabled", "False")
return ret
def __init__(self,folder,collection):
self.folder = folder
self.collection = collection
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
self.instName = instName = os.path.split(instName)[1]
#Load paramFile
self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw')
if self.__paramfile.dataDict["fileID"] != "AcdOptiMetaAnalysis":
raise AcdOptiException_metaAnalysis_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'")
if self.__paramfile.dataDict.getValSingle("instName") != self.instName:
raise AcdOptiException_metaAnalysis_loadFail("instName doesn't match folder name")
self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown"))
self.xVariable = self.__paramfile.dataDict["xVariable"]
self.yVariable = self.__paramfile.dataDict["yVariable"]
if len(self.__paramfile.dataDict.getVals("fVariable")) == 0:
self.__paramfile.dataDict.pushBack("fVariable", "")
self.__paramfile.dataDict.pushBack("fEquals", "")
self.__paramfile.dataDict.pushBack("fLT", "")
self.__paramfile.dataDict.pushBack("fGT", "")
self.__paramfile.write()
self.fVariable = self.__paramfile.dataDict["fVariable"]
def floatOrNone(strIn):
if strIn == "":
return None
else:
return float(strIn)
self.fEquals = floatOrNone(self.__paramfile.dataDict["fEquals"])
self.fGT = floatOrNone(self.__paramfile.dataDict["fGT"])
self.fLT = floatOrNone(self.__paramfile.dataDict["fLT"])
if len(self.__paramfile.dataDict.getVals("targetValue")) > 0:
self.targetValue = float(self.__paramfile.dataDict["targetValue"])
else:
self.targetValue = None
anaData = self.__paramfile.dataDict["anaData"]
self.xArray = []
self.yArray = []
for (x,y) in anaData:
self.xArray.append(float(x))
self.yArray.append(float(y))
#Final init & lockdown checks
if self.lockdown == True:
if self.yVariable == None or self.xVariable == None:
#The x,yArray may still be empty, don't check for this
raise AcdOptiException_metaAnalysis_loadFail("Lockdown, but xVariable='" + self.xVariable + "', yVariable='" + self.yVariable)
self.xArray = np.asarray(self.xArray)
self.yArray = np.asarray(self.yArray)
def __generateSetup_recursiveHelper(setupDict):
print "AcdOptiSolverManager::__generateSetup_recursiveHelper()" # , setupDict=", setupDict
ret = DataDict()
for item in setupDict:
if not DataDict.boolconv(item[1]["enabled"]):
continue
if item[1]["type"] == "dict":
ret.pushBack(item[0], AcdOptiSolverManager.__generateSetup_recursiveHelper(item[1]["children"]))
else:
ret.pushBack(item[0], item[1]["value"])
return ret
def __init__(self, folder, collection):
self.folder = folder
self.collection = collection
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
self.instName = instName = os.path.split(instName)[1]
#Load paramFile
self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw')
if self.__paramfile.dataDict["fileID"] != "AcdOptiDataExtractor":
raise AcdOptiException_dataExtractor_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'")
if self.__paramfile.dataDict.getValSingle("instName") != self.instName:
raise AcdOptiException_dataExtractor_loadFail("instName doesn't match folder name")
self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown"))
try:
self.extractFname = self.__paramfile.dataDict["extractFname"]
except AcdOptiException_dataDict_getValsSingle:
print "AcdOptiDataExtractor::__init__(): Adding extractFname to paramFile"
self.__paramfile.dataDict.pushBack("extractFname", "")
self.__paramfile.write()
self.extractFname = ""
#KeepKeys
self.keepKeys = []
try:
fileKeepKeys = self.__paramfile.dataDict["keepKeys"]
for (k,v) in fileKeepKeys:
assert k == "key"
self.keepKeys.append(v)
except AcdOptiException_dataDict_getValsSingle:
print "AcdOptiDataExtractor::__init__(): Adding keepKeys to paramFile"
self.__paramfile.dataDict.pushBack("keepKeys", DataDict())
self.__paramfile.write()
#Parse keyNames
self.keyNames = []
for (k,v) in self.__paramfile.dataDict["keyNames"]:
assert k == "keyName"
self.keyNames.append(v)
#Parse extractedData
self.dataExtracted = []
for (k,v) in self.__paramfile.dataDict["extractedData"]:
assert k == "dataPoint"
assert isinstance(v,DataDict)
pb = {}
for (k,v) in v:
pb[k] = v
if not k in self.keyNames:
raise AcdOptiException_dataExtractor_loadFail("Key name '" + k + "' not found in self.keyNames")
self.dataExtracted.append(pb)
#Find, load, and attach filters
self.filters = []
for (k,v) in self.__paramfile.dataDict["filters"]:
self.filters.append(AcdOptiDataExtractorFilter.getFilterClass(k,v))
#Plots
self.plots = []
try:
filePlots = self.__paramfile.dataDict["plots"]
for (k,v) in filePlots:
self.plots.append(AcdOptiDataExtractorPlot.getPlotClass(k, self, v))
except AcdOptiException_dataDict_getValsSingle:
self.__paramfile.dataDict.pushBack("plots", DataDict())
self.__paramfile.write()
def __init__(self, folder, scanCollection):
self.folder = folder
self.scanCollection = scanCollection
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
self.instName = instName = os.path.split(instName)[1]
#Load paramFile
self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw')
if self.__paramfile.dataDict["fileID"] != "AcdOptiScan":
raise AcdOptiException_scan_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'")
if self.__paramfile.dataDict.getValSingle("instName") != self.instName:
raise AcdOptiException_scan_loadFail("instName doesn't match folder name")
self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown"))
self.staged = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("staged"))
self.run = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("run"))
self.slaveGeomsDict = self.__paramfile.dataDict["slaveGeoms"]
self.slaveGeoms = []
baseGeomInstance_name = self.__paramfile.dataDict["baseGeomInstance_name"]
if baseGeomInstance_name == "":
self.baseGeomInstance = None
assert len(self.slaveGeomsDict) == 0
else:
self.baseGeomInstance = self.scanCollection.project.geomCollection.geomInstances[baseGeomInstance_name]
if self.__paramfile.dataDict["scanParameter_name"] != "":
#assert self.baseGeomInstance != None #Why?
self.scanParameter_name = self.__paramfile.dataDict["scanParameter_name"]
assert self.scanParameter_name in self.getValidParamNames()
self.scanParameter_range_max = float(self.__paramfile.dataDict["scanParameter_max"])
self.scanParameter_range_min = float(self.__paramfile.dataDict["scanParameter_min"])
self.scanParameter_range_step = float(self.__paramfile.dataDict["scanParameter_step"])
self.generateRange()
try:
self.predict_anaVariable = self.__paramfile.dataDict["predict_anaVariable"]
self.predict_targetValue = self.__paramfile.dataDict["predict_targetValue"]
except AcdOptiException_dataDict_getValsSingle:
self.predict_anaVariable = ""
self.predict_targetValue = ""
self.__paramfile.dataDict.pushBack("predict_anaVariable", "")
self.__paramfile.dataDict.pushBack("predict_targetValue", "")
self.__paramfile.write()
try:
self.predict_a = self.__paramfile.dataDict["predict_a"]
self.predict_b = self.__paramfile.dataDict["predict_b"]
self.predict_x = self.__paramfile.dataDict["predict_x"]
self.predict_r = self.__paramfile.dataDict["predict_r"]
except AcdOptiException_dataDict_getValsSingle:
self.predict_a = ""
self.predict_b = ""
self.predict_x = ""
self.predict_r = ""
self.__paramfile.dataDict.pushBack("predict_a", "")
self.__paramfile.dataDict.pushBack("predict_b", "")
self.__paramfile.dataDict.pushBack("predict_x", "")
self.__paramfile.dataDict.pushBack("predict_r", "")
self.__paramfile.write()
try:
self.predict_ndof=self.__paramfile.dataDict["predict_ndof"]
except AcdOptiException_dataDict_getValsSingle:
self.predict_ndof = ""
self.__paramfile.dataDict.pushBack("predict_ndof","")
self.__paramfile.write()
for (geomName, nothingOfInterest) in self.slaveGeomsDict:
#Mutal referencing
self.slaveGeoms.append(self.scanCollection.project.geomCollection.geomInstances[geomName])
self.slaveGeoms[-1].scanInstances.append(self)
def __init__(self, folder, geometryInstance, meshTemplateCollection):
"""
Loads the meshInstance from the data
from folder and connects it to its template and geometryInstance.
Raises AcdOptiException_meshInstance_loadFail
if something (such as "not a meshInstance folder")
goes wrong.
"""
print "AcdOptiMeshInstance::__init__()"
self.folder = folder
self.geometryInstance = geometryInstance
self.meshTemplateCollection = meshTemplateCollection
if not os.path.isdir(folder):
raise AcdOptiException_meshInstance_loadFail(\
"Folder \"" + folder + "\" does not exist")
#Construct the instance name from folder name
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
instname = os.path.split(instname)[1]
self.instName = instname
#Load paramFile.set
try:
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'rw')
except IOError:
raise AcdOptiException_meshInstance_loadFail(\
"Problem loading file \"paramFile.set\" in folder \"" +\
folder + "\", got an IOError")
if self.__paramFile.dataDict.getValSingle("fileID") != "meshInstance":
raise AcdOptiException_meshInstance_loadFail(
"Wrong fileID in meshInstance.set, got \"" +
self.__paramFile.dataDict.getValSingle("fileID") + "\"")
lock = self.__paramFile.dataDict.getValSingle("lockdown")
if lock == "True":
self.lockdown = True
elif lock == "False":
self.lockdown = False
else:
raise AcdOptiException_meshInstance_loadFail(\
"Invalid content in field \"lockdown\" of file paramFile.set")
meshBad = self.__paramFile.dataDict.getVals("meshBad")
if len(meshBad) == 0:
self.meshBad = False
else:
assert len(meshBad) == 1
self.meshBad = DataDict.boolconv(meshBad[0])
#Check that the geometryInstance is correct
geometryInstance_expectedName = self.__paramFile.dataDict.getValSingle("geomInstance_name")
if geometryInstance.instName != geometryInstance_expectedName:
raise AcdOptiException_meshInstance_loadFail(\
"Excpected name of geometryInstance does not match the one passed")
#Find the MeshTemplate
meshTemplate_expectedName = self.__paramFile.dataDict.getValSingle("meshTemplate_name")
try:
self.meshTemplate = self.meshTemplateCollection.meshTemplates[meshTemplate_expectedName]
self.meshTemplate.registerInstance(self)
except KeyError:
raise AcdOptiException_meshInstance_loadFail("Could not find the meshTemplate \""
+ meshTemplate_expectedName + "\"")
#Find and load template overrides
self.__templateOverrides = {}
try:
templateOverrides_data = self.__paramFile.dataDict.getValSingle("templateOverrides")
except AcdOptiException_dataDict_getValsSingle:
raise AcdOptiException_meshInstance_loadFail\
("Couldn't load templateOverrides from paramFile.set")
if not isinstance(templateOverrides_data,DataDict):
raise AcdOptiException_meshInstance_loadFail\
("templateOverrides from paramFile.set is not a DataDict!")
for (k,v) in zip(templateOverrides_data.keys, templateOverrides_data.vals):
if k in self.__templateOverrides:
raise AcdOptiException_geomCollection_loadFail\
("Double occurrence of key \"" + k + "\" in templateOverrides on file")
if not k in self.meshTemplate.paramDefaults_getKeys():
raise AcdOptiException_meshInstance_loadFail(
"Entry \"" + k + "\" in templateOverrides on file has no match in the template")
self.__templateOverrides[k] = v
#Find subfolders and check if they are runConfigs
if not os.path.isdir(os.path.join(self.folder, "runConfigs")):
raise AcdOptiException_meshInstance_loadFail("Could not find runConfigs folder")
self.runConfigs = {}
for d in os.listdir(os.path.join(self.folder,"runConfigs")):
dAbs = os.path.abspath(os.path.join(self.folder,"runConfigs",d))
if not os.path.isdir(dAbs):
#Skip files etc.
continue
#try:
self.runConfigs[d] = AcdOptiRunConfig(dAbs,self)
