def createNew(folder):
"""
Creates a new empty geomInstance in a
not previously existing folder.
Geometry instance name will be the same as the folder name.
Raises AcdOptiException_geomInstance_createFail
is something goes wrong (such as "Folder already exists")
"""
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
#Create the folder
if os.path.isdir(folder):
raise AcdOptiException_geomInstance_createFail("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create paramFile.set file
paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "geomInstance")
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.dataDict.pushBack("templateOverrides", DataDict())
#paramFile.dataDict.pushBack("scanInstance_name", "")
paramFile.write()
#Create folder for meshInstance's
os.mkdir(os.path.join(folder, AcdOptiGeometryInstance.meshInstanceFolderName))
def createNew(folder):
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
if os.path.isdir(folder):
raise AcdOptiException_metaAnalysis_createFail("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create paramfile
paramFile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "AcdOptiMetaAnalysis")
paramFile.dataDict.pushBack("instName", instName)
paramFile.dataDict.pushBack("xVariable", "")
paramFile.dataDict.pushBack("yVariable", "")
paramFile.dataDict.pushBack("fVariable", "")
paramFile.dataDict.pushBack("fEquals", "")
paramFile.dataDict.pushBack("fLT", "")
paramFile.dataDict.pushBack("fGT", "")
paramFile.dataDict.pushBack("anaData", DataDict()) #(key, value) = (x,y)
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.write()
def createNew(folder):
"""
Creates a new empty meshTemplate in a
not previously existing folder.
Folder name should be the same as geometry instance name.
Raises AcdOptiException_meshTemplate_createFail
is something goes wrong (such as "Folder already exists")
"""
#Construct the instance name from folder
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
instname = os.path.split(instname)[1]
if os.path.isdir(folder):
raise AcdOptiException_meshTemplate_createFail ("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create the paramFile
paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "meshTemplateParamFile")
paramFile.dataDict.pushBack("instName", instname)
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.dataDict.pushBack("paramDefaults", DataDict())
paramFile.write()
#Default empty template file
AcdOptiCubitTemplateFile.createEmpty(os.path.join(folder,AcdOptiMeshTemplate.meshTemplateFile_name))
def createNew(folder):
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
if os.path.isdir(folder):
raise AcdOptiException_dataExtractor_createFail("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create paramfile
paramFile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "AcdOptiDataExtractor")
paramFile.dataDict.pushBack("instName", instName)
paramFile.dataDict.pushBack("extractFname", "")
paramFile.dataDict.pushBack("keyNames", DataDict())
paramFile.dataDict.pushBack("extractedData", DataDict())
paramFile.dataDict.pushBack("filters", DataDict())
paramFile.dataDict.pushBack("plots", DataDict())
paramFile.dataDict.pushBack("keepKeys", DataDict())
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.write()
def createNew(type, folder, name=None):
"""
Prepares a new AcdOptiSolverManager by creating the metaFile.
Input:
- type is the name of the type ("omega3P" etc.) wanted
- folder is where this instance should be created
- name is the name wanted for this instance.
Set to None to use default name.
Name is also the filename of the solver setup file generated.
Returns the name used.
"""
print "AcdOptiSolverSetupManager::createNew(), type=" + type + ", name=" + str(name)
# Check that the type is valid
if not type in AcdOptiSolverManager.getTypes():
raise TypeError("type '" + type + "' is invalid")
# Load the type definition
typeFile = AcdOptiFileParser_simple(os.path.join(solverSetupResourcesPath, type + ".set"), "r")
if typeFile.dataDict["fileID"] != "SolverSetupTemplate":
raise AcdOptiException_solverSetup_createFail("Got fileID ='" + str(typeFile.dataDict["fileID"]) + "'")
# Check that everything is OK with the typeFile
# TODO!
# Read the typeFile
if not name:
name = typeFile.dataDict.getValSingle("fileNameDefault")
# Generate the metaFile
metaFileName = os.path.join(folder, name + ".meta")
# First: check that the name is not in use
if os.path.isfile(os.path.join(folder, metaFileName)):
raise AcdOptiException_solverSetup_createFail_nameTaken(
"Name already in use, couldn't create metaFile", name
)
metaFile = AcdOptiFileParser_simple(os.path.join(folder, metaFileName), "w")
# Setup the basic structure of the metaFile
metaFile.dataDict.pushBack("fileID", "SolverManager")
metaFile.dataDict.pushBack("name", name)
metaFile.dataDict.pushBack("fileFormat", typeFile.dataDict["fileFormat"])
metaFile.dataDict.pushBack("type", typeFile.dataDict["type"])
# Generate the options (which in itself is a childDict)
metaFile.dataDict.pushBack(
"options", AcdOptiSolverManager.__genMetaOptions(typeFile.dataDict.getValSingle("options"))
)
metaFile.write()
return name
def __init__(self,folder,collection):
self.folder = folder
self.collection = collection
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
self.instName = instName = os.path.split(instName)[1]
#Load paramFile
self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw')
if self.__paramfile.dataDict["fileID"] != "AcdOptiMetaAnalysis":
raise AcdOptiException_metaAnalysis_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'")
if self.__paramfile.dataDict.getValSingle("instName") != self.instName:
raise AcdOptiException_metaAnalysis_loadFail("instName doesn't match folder name")
self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown"))
self.xVariable = self.__paramfile.dataDict["xVariable"]
self.yVariable = self.__paramfile.dataDict["yVariable"]
if len(self.__paramfile.dataDict.getVals("fVariable")) == 0:
self.__paramfile.dataDict.pushBack("fVariable", "")
self.__paramfile.dataDict.pushBack("fEquals", "")
self.__paramfile.dataDict.pushBack("fLT", "")
self.__paramfile.dataDict.pushBack("fGT", "")
self.__paramfile.write()
self.fVariable = self.__paramfile.dataDict["fVariable"]
def floatOrNone(strIn):
if strIn == "":
return None
else:
return float(strIn)
self.fEquals = floatOrNone(self.__paramfile.dataDict["fEquals"])
self.fGT = floatOrNone(self.__paramfile.dataDict["fGT"])
self.fLT = floatOrNone(self.__paramfile.dataDict["fLT"])
if len(self.__paramfile.dataDict.getVals("targetValue")) > 0:
self.targetValue = float(self.__paramfile.dataDict["targetValue"])
else:
self.targetValue = None
anaData = self.__paramfile.dataDict["anaData"]
self.xArray = []
self.yArray = []
for (x,y) in anaData:
self.xArray.append(float(x))
self.yArray.append(float(y))
#Final init & lockdown checks
if self.lockdown == True:
if self.yVariable == None or self.xVariable == None:
#The x,yArray may still be empty, don't check for this
raise AcdOptiException_metaAnalysis_loadFail("Lockdown, but xVariable='" + self.xVariable + "', yVariable='" + self.yVariable)
self.xArray = np.asarray(self.xArray)
self.yArray = np.asarray(self.yArray)
def createNew(folder):
"""
Sets up the basic structure inside
a given folder, which is created.
"""
#Create the directory
os.mkdir(folder)
#File that holds the parameters and their default values
paramFile = AcdOptiFileParser_simple(\
os.path.join(folder,"paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "geomCollectionParamFile")
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.dataDict.pushBack("paramDefaults",DataDict())
paramFile.write()
#Default empty template file
AcdOptiCubitTemplateFile.createEmpty(os.path.join(folder,AcdOptiGeometryCollection.geomTemplateFile_name))
def __init__(self, folder):
self.folder = folder
#Construct the instance name from folder
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
self.instName = instname = os.path.split(instname)[1]
#Load the param file
try:
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(self.folder, "paramFile.set"), 'rw')
except IOError:
raise AcdOptiException_meshTemplate_loadFail("File paramFile.set not found")
if self.__paramFile.dataDict.getValSingle("fileID")\
!= "meshTemplateParamFile":
raise AcdOptiException_meshTemplate_loadFail\
("Wrong fileID, got \""\
+ self.__paramFile.dataDict.getValSingle("fileID")\
+ "\" while loading paramFile")
if self.__paramFile.dataDict.getValSingle("instName")\
!= instname:
raise AcdOptiException_meshTemplate_loadFail("templateName doesn't match folder name")
lock = self.__paramFile.dataDict.getValSingle("lockdown")
if lock == "True":
self.lockdown = True
elif lock == "False":
self.lockdown = False
else:
raise AcdOptiException_meshTemplate_loadFail(\
"Invalid content in field \"lockdown\" of file paramFile.set, got'"+lock+"'")
#Load the default parameters
self.__paramDefaults = {}
try:
paramDefaults_data = self.__paramFile.dataDict.getValSingle("paramDefaults")
except AcdOptiException_dataDict_getValsSingle:
raise AcdOptiException_meshTemplate_loadFail\
("Couldn't load paramDefaults from file paramFile.set")
if not isinstance(paramDefaults_data,DataDict):
raise AcdOptiException_meshTemplate_loadFail\
("paramDefaults from paramFile is not a DataDict!")
for (k,v) in zip(paramDefaults_data.keys, paramDefaults_data.vals):
if k in self.__paramDefaults:
raise AcdOptiException_meshTemplate_loadFail\
("Double occurrence of key \"" + k + "\" in paramFile")
self.__paramDefaults[k] = v
#Load the template file
self.meshTemplateFile = AcdOptiCubitTemplateFile(os.path.join(folder,self.meshTemplateFile_name))
#Initialize __meshInstances
self.__meshInstances = []
def __init__(self, name, folder):
"""
Initialize a solverManager.
- Name: Name of this instance (used to find the metaFile and name of the created setup file)
- folder: Where the metaFile lives, and where the setup file is created.
"""
print "AcdOptiSolverManager::__init__(), name='" + name + "', folder='" + folder + "'"
self.folder = folder
self.name = name
if not os.path.isdir(self.folder):
raise AcdOptiException_solverSetup_loadFail("Subfolder 'stage' is missing")
# Load the metaFile
self.metaSetupFilePath = metaFileName = os.path.join(self.folder, name + ".meta")
self.__metaSetupFile = AcdOptiFileParser_simple(metaFileName, "rw")
if self.__metaSetupFile.dataDict["fileID"] != "SolverManager":
if self.__metaSetupFile.dataDict["fileID"] == "SolverSetup":
print "\t WARNING: Old-style file detected, fileID='SolverSetup'"
print "\t Will change it to new-style 'solverManager'"
self.__metaSetupFile.dataDict.setValSingle("fileID", "SolverManager")
self.__metaSetupFile.write()
else:
raise AcdOptiException_solverSetup_loadFail(
"Wrong fileID in metaSetupFile '" + self.metaSetupFilePath + "'"
)
if self.name != self.__metaSetupFile.dataDict["name"]:
raise AcdOptiException_solverSetup_loadFail(
"name in metaSetupFile '"
+ self.__metaSetupFile.dataDict["name"]
+ "' doesn't match provided argument '"
+ self.name
+ "'"
)
self.setupFileFormat = self.__metaSetupFile.dataDict.getValSingle("fileFormat")
if self.setupFileFormat == "simple":
self.setupFileFormat = AcdOptiFileParser_simple
elif self.setupFileFormat == "Lua":
self.setupFileFormat = AcdOptiFileParser_Lua
elif self.setupFileFormat == "KVC":
self.setupFileFormat = AcdOptiFileParser_KVC
else:
raise ValueError("Unknown setupFileFormat encountered, setupFileFormat='" + self.setupFileFormat + "'")
self.type = self.__metaSetupFile.dataDict["type"]
self.metaSetup = self.__metaSetupFile.dataDict.getValSingle("options")
self.fileName = os.path.join(self.folder, self.name)
def createNew(folder):
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
if os.path.isdir(folder):
raise AcdOptiException_scan_createFail("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create paramfile
paramFile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "AcdOptiScan")
paramFile.dataDict.pushBack("instName", instName)
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.dataDict.pushBack("staged", "False")
paramFile.dataDict.pushBack("run", "False")
paramFile.dataDict.pushBack("baseGeomInstance_name", "")
paramFile.dataDict.pushBack("slaveGeoms", DataDict())
paramFile.dataDict.pushBack("scanParameter_name", "")
paramFile.dataDict.pushBack("scanParameter_max", "")
paramFile.dataDict.pushBack("scanParameter_min","")
paramFile.dataDict.pushBack("scanParameter_step","")
paramFile.dataDict.pushBack("predict_anaVariable", "")
paramFile.dataDict.pushBack("predict_targetValue", "")
paramFile.dataDict.pushBack("predict_a", "")
paramFile.dataDict.pushBack("predict_b", "")
paramFile.dataDict.pushBack("predict_x", "")
paramFile.dataDict.pushBack("predict_r", "")
paramFile.dataDict.pushBack("predict_ndof", "")
paramFile.write()
def createNew(folder, type):
#Create the settings file
paramFile = AcdOptiFileParser_simple(os.path.join(folder,"paramFile_acdOptiRunner_Hopper.set"),"w")
paramFile.dataDict.pushBack("fileID", "AcdOptiRunner_Hopper")
paramFile.dataDict.pushBack("remoteJobID", "")
#Set default torque meta stuff
torqueMeta = paramFile.dataDict.pushBack("TorqueMeta", DataDict())
torqueMeta.pushBack("queue", "regular")
torqueMeta.pushBack("walltime", "00:59:00")
torqueMeta.pushBack("repo", "m349")
torqueMeta.pushBack("importVars", "True")
#Create a datastructure for storing aprun jobs
jobs = paramFile.dataDict.pushBack("jobs", DataDict())
# Each aprun job has the following fields:
# - aprun: Boolean, true in the case of aprun jobs
# - command: Command to run
# - commandArgs: Arguments to pass to the executable (such as name of input file)
# - tasks: Number of MPI tasks, -n. Essential!
# - tasksNode: Number of MPI tasks pr. node, -N. Optional.
# - tasksNuma: Number of MPI tasks pr. NUMA node, -S. Optional.
# If the aprun flag is False, then only command is used (but all keys should be present!)
# Optional args should also be present. Set to "-1" to disable.
#This command is always needed.
cdpbs = jobs.pushBack("cdPBS", DataDict())
cdpbs.pushBack("aprun", "False")
cdpbs.pushBack("command", "cd $PBS_O_WORKDIR")
cdpbs.pushBack("commandArgs", "")
cdpbs.pushBack("tasks", "-1")
cdpbs.pushBack("tasksNode", "-1")
cdpbs.pushBack("tasksNuma", "-1")
paramFile.write()
def __init__(self,runConfig):
self.runConfig = runConfig
self.folder = self.runConfig.folder
self.__paramFile = AcdOptiFileParser_simple(os.path.join(self.folder,"paramFile_acdOptiRunner_Hopper.set"), 'rw')
if not self.__paramFile.dataDict["fileID"] == "AcdOptiRunner_Hopper":
raise AcdOptiException_optiRunner_loadFail("Wrong fileID, got'"+self.__paramFile.dataDict["fileID"] + "'")
self.remoteJobID = self.__paramFile.dataDict["remoteJobID"]
if self.remoteJobID == "":
self.remoteJobID = None
if self.remoteJobID != None and not self.runConfig.status.startswith("remote::"):
print "WARNING: Found remoteJobID, but status='" + self.runConfig.status + "'"
#raise AcdOptiException_optiRunner_loadFail("Found remoteJobID, but status='" + self.runConfig.status + "'")
elif self.remoteJobID == None and (self.runConfig.status == "remote::queued" or self.runConfig.status == "remote::running"):
raise AcdOptiException_optiRunner_loadFail("Did not find remoteJobID, but status='" + self.runConfig.status + "'")
self.speedSSH = SSHspeedtransfer(self.hostname, AcdOptiSettings().getSetting("hopperUser"))
class AcdOptiScan:
"""
Class representing a geometry parameter scan.
This class can "own" geometryInstances, just like a geometryCollection
"""
folder = None
scanCollection = None
__paramfile = None
instName = None
lockdown = None #irreversible lockdown on parameters, happens after createScan()
staged = None #2'nd "lockdown", indicates that geoms has been staged.
run = None #3'rd, indicates that geoms has been uploaded&ran.
baseGeomInstance = None
slaveGeoms = None
slaveGeomsDict = None
scanParameter_name = None #Name of the parameter that should be scanned.
scanParameter_range_max = None # If this is set, so should the range stuff.
scanParameter_range_min = None
scanParameter_range_step = None #Step length
scanParameter_range = None #List with wanted values of the scan
predict_anaVariable = None
predict_targetValue = None
predict_a = None #y=ax+b
predict_b = None
predict_x = None #predicted x s.t. y=target
predict_r = None #sqrt(R^2)
predict_ndof = None
def __init__(self, folder, scanCollection):
self.folder = folder
self.scanCollection = scanCollection
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
self.instName = instName = os.path.split(instName)[1]
#Load paramFile
self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw')
if self.__paramfile.dataDict["fileID"] != "AcdOptiScan":
raise AcdOptiException_scan_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'")
if self.__paramfile.dataDict.getValSingle("instName") != self.instName:
raise AcdOptiException_scan_loadFail("instName doesn't match folder name")
self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown"))
self.staged = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("staged"))
self.run = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("run"))
self.slaveGeomsDict = self.__paramfile.dataDict["slaveGeoms"]
self.slaveGeoms = []
baseGeomInstance_name = self.__paramfile.dataDict["baseGeomInstance_name"]
if baseGeomInstance_name == "":
self.baseGeomInstance = None
assert len(self.slaveGeomsDict) == 0
else:
self.baseGeomInstance = self.scanCollection.project.geomCollection.geomInstances[baseGeomInstance_name]
if self.__paramfile.dataDict["scanParameter_name"] != "":
#assert self.baseGeomInstance != None #Why?
self.scanParameter_name = self.__paramfile.dataDict["scanParameter_name"]
assert self.scanParameter_name in self.getValidParamNames()
self.scanParameter_range_max = float(self.__paramfile.dataDict["scanParameter_max"])
self.scanParameter_range_min = float(self.__paramfile.dataDict["scanParameter_min"])
self.scanParameter_range_step = float(self.__paramfile.dataDict["scanParameter_step"])
self.generateRange()
try:
self.predict_anaVariable = self.__paramfile.dataDict["predict_anaVariable"]
self.predict_targetValue = self.__paramfile.dataDict["predict_targetValue"]
except AcdOptiException_dataDict_getValsSingle:
self.predict_anaVariable = ""
self.predict_targetValue = ""
self.__paramfile.dataDict.pushBack("predict_anaVariable", "")
self.__paramfile.dataDict.pushBack("predict_targetValue", "")
self.__paramfile.write()
try:
self.predict_a = self.__paramfile.dataDict["predict_a"]
self.predict_b = self.__paramfile.dataDict["predict_b"]
self.predict_x = self.__paramfile.dataDict["predict_x"]
self.predict_r = self.__paramfile.dataDict["predict_r"]
except AcdOptiException_dataDict_getValsSingle:
self.predict_a = ""
self.predict_b = ""
self.predict_x = ""
self.predict_r = ""
self.__paramfile.dataDict.pushBack("predict_a", "")
self.__paramfile.dataDict.pushBack("predict_b", "")
self.__paramfile.dataDict.pushBack("predict_x", "")
self.__paramfile.dataDict.pushBack("predict_r", "")
self.__paramfile.write()
try:
self.predict_ndof=self.__paramfile.dataDict["predict_ndof"]
except AcdOptiException_dataDict_getValsSingle:
self.predict_ndof = ""
self.__paramfile.dataDict.pushBack("predict_ndof","")
self.__paramfile.write()
for (geomName, nothingOfInterest) in self.slaveGeomsDict:
#Mutal referencing
self.slaveGeoms.append(self.scanCollection.project.geomCollection.geomInstances[geomName])
self.slaveGeoms[-1].scanInstances.append(self)
def generateRange(self):
"Generate the scan range"
if self.scanParameter_range_max == None or self.scanParameter_range_min == None or self.scanParameter_range_step == None:
raise AcdOptiException_scan_generateRangeFail("Range parameters not set, max='" + str(self.scanParameter_range_max) +\
"', min='" + str(self.scanParameter_range_min) + "', step='" + str(self.scanParameter_range_step) + "'")
# if self.lockdown:
# raise AcdOptiException_scan_generateRangeFail("Can't generate range: In lockdown!")
#
self.scanParameter_range = np.arange(self.scanParameter_range_min,\
self.scanParameter_range_max,\
self.scanParameter_range_step)
def write(self):
if self.baseGeomInstance != None:
self.__paramfile.dataDict.setValSingle("baseGeomInstance_name", self.baseGeomInstance.instName)
else:
self.__paramfile.dataDict.setValSingle("baseGeomInstance_name", "")
if self.scanParameter_name != None:
self.__paramfile.dataDict.setValSingle("scanParameter_name", self.scanParameter_name)
assert self.scanParameter_range_max != None and self.scanParameter_range_min != None and self.scanParameter_range_step != None
self.__paramfile.dataDict.setValSingle("scanParameter_max", str(self.scanParameter_range_max))
self.__paramfile.dataDict.setValSingle("scanParameter_min", str(self.scanParameter_range_min))
self.__paramfile.dataDict.setValSingle("scanParameter_step", str(self.scanParameter_range_step))
else:
self.__paramfile.dataDict.setValSingle("scanParameter_name", "")
assert self.scanParameter_range_max == None and self.scanParameter_range_min == None and self.scanParameter_range_step == None
self.__paramfile.dataDict.setValSingle("scanParameter_max", "")
self.__paramfile.dataDict.setValSingle("scanParameter_min", "")
self.__paramfile.dataDict.setValSingle("scanParameter_step", "")
self.slaveGeomsDict.clear()
if len(self.slaveGeoms) != 0:
assert self.baseGeomInstance != None
for geom in self.slaveGeoms:
self.slaveGeomsDict.pushBack(geom.instName, "")
self.__paramfile.dataDict.setValSingle("lockdown", str(self.lockdown))
self.__paramfile.dataDict.setValSingle("staged" , str(self.staged))
self.__paramfile.dataDict.setValSingle("run" , str(self.run))
self.__paramfile.dataDict.setValSingle("predict_anaVariable", self.predict_anaVariable)
self.__paramfile.dataDict.setValSingle("predict_targetValue", self.predict_targetValue)
self.__paramfile.dataDict.setValSingle("predict_a", self.predict_a)
self.__paramfile.dataDict.setValSingle("predict_b", self.predict_b)
self.__paramfile.dataDict.setValSingle("predict_x", self.predict_x)
self.__paramfile.dataDict.setValSingle("predict_r", self.predict_r)
self.__paramfile.dataDict.setValSingle("predict_ndof", self.predict_ndof)
self.__paramfile.write()
def getValidParamNames(self):
"Selects the name of the scan parameter among the parameters in the geometry"
return self.scanCollection.project.geomCollection.paramDefaults_getKeys()
def createScan(self):
"""
Using the data in baseGeomInstance_name and the range,
create the scan. A function that will be called on each iteration may be provided.
If a common error occurs, a AcdOptiException_scan_scanFail with the error message of interest is raised.
"""
#Check that we are ready to scan
if self.lockdown:
raise AcdOptiException_scan_scanFail("Scan already created.")
if not self.scanParameter_name in self.getValidParamNames():
raise AcdOptiException_scan_scanFail("No (valid) scan parameter selected.")
if self.scanParameter_range_min == None or self.scanParameter_range_max == None or self.scanParameter_range_step == None:
raise AcdOptiException_scan_scanFail("Scan range not ready.")
self.generateRange()
if len(self.scanParameter_range) == 0:
raise AcdOptiException_scan_scanFail("Scan range empty.")
if not self.baseGeomInstance in self.scanCollection.project.geomCollection.geomInstances.values():
raise AcdOptiException_scan_scanFail("No (valid) geomInstance selected")
for val in self.scanParameter_range:
self.addPoint(val)
# oldName = self.baseGeomInstance.instName
# newName = oldName + "--scan--" + self.scanParameter_name + str(val) #Separators messes things up..
# #Check if this already exists:
# newGeom = None
# if newName in self.scanCollection.project.geomCollection.geomInstances:
# #It's already there!
# newGeom = self.scanCollection.project.geomCollection.geomInstances[newName]
# newGeom.scanInstances.append(self)
# #TODO: Now assuming that the name is an unique identifier for a configuration.
# # This might not be safe - maybe add some kind of lockdown?
# else:
# newGeom = self.scanCollection.project.geomCollection.cloneGeomInstance(oldName,newName)
# newGeom.templateOverrides_insert(self.scanParameter_name, str(val))
# newGeom.scanInstances.append(self)
# newGeom.write()
# self.slaveGeoms.append(newGeom)
self.lockdown = True
self.write()
def stageAll(self, progressCallback=None):
"""
Stage all geoms/meshes/runConfigs in self.slaveGeoms
"""
print "AcdOptiScan::stageAll()"
if not self.lockdown:
raise AcdOptiException_scan_stageFail("Lockdown not set, not ready to stage!")
for geom in self.slaveGeoms:
for mesh in geom.meshInsts.values():
for rc in mesh.runConfigs.values():
if rc.status == "finished":
continue #In case of geom reuse
try:
rc.stage()
except AcdOptiException_runConfig_stageError:
print "Staging of rc failed - skipping to the next one."
if progressCallback != None:
print "AcdOptiScan::stageAll() : progressCallback()"
progressCallback()
self.staged = True
def runScan(self):
"""
Upload and run all runConfigs
"""
print "AcdOptiScan::RunScan()"
if not self.staged:
raise AcdOptiException_scan_runFail("Not staged yet!")
for geom in self.slaveGeoms:
for mesh in geom.meshInsts.values():
for rc in mesh.runConfigs.values():
if rc.status != "staged":
continue #In case of geom reuse or nongenerated mesh'es
rc.upload()
rc.run()
self.run = True
def refreshAndDownload(self):
"""
Refresh the status on all the runConfigs, download results where finished
"""
print "AcdOptiScan::refreshAndDownload()"
if self.run != True:
raise AcdOptiException_scan_refreshDownloadFail("Not yet ran, run != True")
for geom in self.slaveGeoms:
for mesh in geom.meshInsts.values():
for rc in mesh.runConfigs.values():
if rc.status.startswith("remote::"):
rc.refreshStatus()
#TODO: Handle possible local runner?
if rc.status == "remote::finished":
rc.getRemote()
def runAnalysis(self):
"""
For all finished runConfigs, run the defined analysis.
"""
print "AcdOptiScan::runAnalysis()"
if self.run != True:
raise AcdOptiException_scan_analysisFail("Not yet ran, run != True")
for geom in self.slaveGeoms:
for mesh in geom.meshInsts.values():
for rc in mesh.runConfigs.values():
if rc.status.startswith("finished"):
for analysis in rc.analysis.values():
if not analysis.lockdown:
try:
analysis.runAnalysis()
except AcdOptiException_analysis_runAnalysis as e:
print "Error in analysis '" + analysis.instName + "': '" + str(e.args) + "', skipping!"
def predictCorrectValue(self):
"""
Fit the some analysis result versus the scanned parameter,
and predict the value of this parameter that yields
the analysis result equals a target value.
Returns: predicted parameter value, sum of squared residuals in fit.
"""
anaVariable = self.predict_anaVariable
targetValue = float(self.predict_targetValue)
#Get fit data
x = []
y = []
anaVarSplit = anaVariable.split(".")
#for geom in self.slaveGeoms:
for geom in self.scanCollection.project.geomCollection.geomInstances.itervalues():
#skip geometries which differs in more than one parameter
goodGeom = True
for param in self.scanCollection.project.geomCollection.paramDefaults_getKeys():
if param == self.scanParameter_name:
continue
if param in self.baseGeomInstance.templateOverrides_getKeys():
paramVal = self.baseGeomInstance.templateOverrides_get(param)
else:
paramVal = self.scanCollection.project.geomCollection.paramDefaults_get(param)
if param in geom.templateOverrides_getKeys():
paramVal2 = geom.templateOverrides_get(param)
else:
paramVal2 = self.scanCollection.project.geomCollection.paramDefaults_get(param)
if paramVal != paramVal2:
goodGeom = False
break
if not goodGeom:
continue #Skip this geometry
#Baseline geom may be included; just skip'it (including it makes it neccessary to rewrite the code below)
if not self.scanParameter_name in geom.templateOverrides_getKeys():
continue
thisX=float(geom.templateOverrides_get(self.scanParameter_name))
for mesh in geom.meshInsts.itervalues():
for rc in mesh.runConfigs.itervalues():
for ana in rc.analysis.itervalues():
if ana.instName == anaVarSplit[0]:
#Recursive function to find the analysis result
#Uses the usual key.key[idx].key.... syntax
#If not [idx] present, assume idx=0
#If not found, return None
def dictRecDig(avsRemaining, dictBranch):
avsParsed = re.match(r'(\w)+[(\d+)]',avsRemaining[0])
if avsParsed != None:
nextName = avsParsed.group(1)
nextNumber = int(avsParsed.group(2))
else:
nextName = avsRemaining[0]
nextNumber = 0
try:
nextBranch = dictBranch.getVals(nextName)[nextNumber]
except IndexError:
print "WARNING in dictRecDig(): Key '" + avsRemaining[0] + "' not found"
return None
if isinstance(nextBranch,DataDict):
if len(avsRemaining) == 1:
print "WARNING in dictRecDig(): More depth than keys"
return None
return dictRecDig(avsRemaining[1:],nextBranch)
else:
if len(avsRemaining) > 1:
print "WARNING in dictRecDig(): More keys than depth"
return None
return float(nextBranch)
thisY = dictRecDig(anaVarSplit[1:],ana.exportResults)
if thisY != None:
x.append(thisX)
y.append(thisY)
print "X=", x
print "Y=", y
#Fit function y = ax+b
obs = len(y)
ndof = obs-2
if ndof < 0:
raise AcdOptiException_scan_predictFail("ndof = " + str(ndof) + " < 0")
self.predict_ndof=str(ndof)
H = np.ones([obs,2])
for i in xrange(obs):
H[i,1] = x[i]
project = np.dot(np.linalg.inv(np.dot(H.transpose(),H)), H.transpose())
y = np.asarray(y)
theta = np.dot(project,y)
print "THETA=(y0,dy/dx)=", theta
self.predict_a=str(theta[1])
self.predict_b=str(theta[0])
#Find x such that y=target
x_predicted = (targetValue-theta[0])/theta[1]
print "X_PREDICTED=",x_predicted
self.predict_x = str(x_predicted)
#Calculate R^2
R2 = 0.0
for i in xrange(obs):
yhat = theta[0] + theta[1]*x[i]
print "Y YHAT = ", y[i], yhat, yhat-y[i]
R2 += (yhat-y[i])**2
print "sqrt(R**2)=", np.sqrt(R2)
self.predict_r = str(np.sqrt(R2))
self.write()
def addPoint(self,parameterValue):
"""
Add a point to the scan with the given parameter value
"""
oldName = self.baseGeomInstance.instName
newName = oldName + "--scan--" + self.scanParameter_name + str(parameterValue) #Separators messes things up..
#Check if this already exists:
newGeom = None
if newName in self.scanCollection.project.geomCollection.geomInstances:
#It's already there!
newGeom = self.scanCollection.project.geomCollection.geomInstances[newName]
newGeom.scanInstances.append(self)
#TODO: Now assuming that the name is an unique identifier for a configuration.
# This might not be safe - maybe add some kind of lockdown?
else:
newGeom = self.scanCollection.project.geomCollection.cloneGeomInstance(oldName,newName)
newGeom.templateOverrides_insert(self.scanParameter_name, str(parameterValue))
newGeom.scanInstances.append(self)
newGeom.write()
self.slaveGeoms.append(newGeom)
@staticmethod
def createNew(folder):
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
if os.path.isdir(folder):
raise AcdOptiException_scan_createFail("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create paramfile
paramFile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "AcdOptiScan")
paramFile.dataDict.pushBack("instName", instName)
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.dataDict.pushBack("staged", "False")
paramFile.dataDict.pushBack("run", "False")
paramFile.dataDict.pushBack("baseGeomInstance_name", "")
paramFile.dataDict.pushBack("slaveGeoms", DataDict())
paramFile.dataDict.pushBack("scanParameter_name", "")
paramFile.dataDict.pushBack("scanParameter_max", "")
paramFile.dataDict.pushBack("scanParameter_min","")
paramFile.dataDict.pushBack("scanParameter_step","")
paramFile.dataDict.pushBack("predict_anaVariable", "")
paramFile.dataDict.pushBack("predict_targetValue", "")
paramFile.dataDict.pushBack("predict_a", "")
paramFile.dataDict.pushBack("predict_b", "")
paramFile.dataDict.pushBack("predict_x", "")
paramFile.dataDict.pushBack("predict_r", "")
paramFile.dataDict.pushBack("predict_ndof", "")
paramFile.write()
def __init__(self, folder, template):
"""
Loads the geomInstance from the data
from folder.
Raises AcdOptiException_geomInstance_loadFail
if something (such as "not a geomInstance folder")
goes wrong.
"""
self.folder = folder
self.template = template
#Check that folder exists
if not os.path.isdir(folder):
raise AcdOptiException_geomInstance_loadFail(\
"Folder \"" + folder + "\" does not exist")
#Construct the instance name from folder name
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
self.instName = instName
#Load paramFile.set
try:
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'rw')
except IOError as e:
raise AcdOptiException_geomInstance_loadFail(\
"Problem loading file \"paramFile.set\" in folder \"" +\
folder + "\", got an IOError")
if self.__paramFile.dataDict.getValSingle("fileID") != "geomInstance":
raise AcdOptiException_geomInstance_loadFail(
"Wrong fileID in geomInstance.set, got \"" +
self.__paramFile.dataDict.getValSingle("fileID") + "\"")
lock = self.__paramFile.dataDict.getValSingle("lockdown")
if lock == "True":
self.lockdown = True
elif lock == "False":
self.lockdown = False
else:
raise AcdOptiException_geomInstance_loadFail(\
"Invalid content in field \"lockdown\" of file paramFile.set")
#Find and load template overrides
self.__templateOverrides = {}
try:
templateOverrides_data = self.__paramFile.dataDict.getValSingle("templateOverrides")
except AcdOptiException_dataDict_getValsSingle:
raise AcdOptiException_geomInstance_loadFail\
("Couldn't load templateOverrides from paramFile.set")
if not isinstance(templateOverrides_data,DataDict):
raise AcdOptiException_geomInstance_loadFail\
("templateOverrides from paramFile.set is not a DataDict!")
for (k,v) in zip(templateOverrides_data.keys, templateOverrides_data.vals):
if k in self.__templateOverrides:
raise AcdOptiException_geomInstance_loadFail\
("Double occurrence of key \"" + k + "\" in templateOverrides on file")
if not k in self.template.paramDefaults_getKeys():
raise AcdOptiException_geomInstance_loadFail(
"Entry \"" + k + "\" in templateOverrides on file has no match in the template")
self.__templateOverrides[k] = v
#Are there any mesh instances?
self.meshInsts = {}
for d in os.listdir(os.path.join(self.folder,self.meshInstanceFolderName)):
dAbs = os.path.abspath(os.path.join(folder,self.meshInstanceFolderName,d))
if not os.path.isdir(dAbs):
#Skip files etc.
continue
#try:
self.meshInsts[d] = AcdOptiMeshInstance(dAbs, self, self.template.project.meshTemplateCollection)
#except AcdOptiException_meshInstance_loadFail as e:
# raise AcdOptiException_geomCollection_loadFail(\
# "Problem loading mesh instance \"" + dAbs + "\", got error \"" + str(e) + "\"")
#Scan instance name
#self.scanInstance_name = self.__paramFile.dataDict["scanInstance_name"]
self.scanInstances = []
class AcdOptiDataExtractor:
"""
This class extracts data from analysis, geoms etc.
This is presented in a FLAT table.
"""
instName = None
folder = None
collection = None
__paramfile = None
extractFname = None
dataExtracted = None
keyNames = None
filters = None
plots = None
keepKeys = None
lockdown = None
def __init__(self, folder, collection):
self.folder = folder
self.collection = collection
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
self.instName = instName = os.path.split(instName)[1]
#Load paramFile
self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw')
if self.__paramfile.dataDict["fileID"] != "AcdOptiDataExtractor":
raise AcdOptiException_dataExtractor_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'")
if self.__paramfile.dataDict.getValSingle("instName") != self.instName:
raise AcdOptiException_dataExtractor_loadFail("instName doesn't match folder name")
self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown"))
try:
self.extractFname = self.__paramfile.dataDict["extractFname"]
except AcdOptiException_dataDict_getValsSingle:
print "AcdOptiDataExtractor::__init__(): Adding extractFname to paramFile"
self.__paramfile.dataDict.pushBack("extractFname", "")
self.__paramfile.write()
self.extractFname = ""
#KeepKeys
self.keepKeys = []
try:
fileKeepKeys = self.__paramfile.dataDict["keepKeys"]
for (k,v) in fileKeepKeys:
assert k == "key"
self.keepKeys.append(v)
except AcdOptiException_dataDict_getValsSingle:
print "AcdOptiDataExtractor::__init__(): Adding keepKeys to paramFile"
self.__paramfile.dataDict.pushBack("keepKeys", DataDict())
self.__paramfile.write()
#Parse keyNames
self.keyNames = []
for (k,v) in self.__paramfile.dataDict["keyNames"]:
assert k == "keyName"
self.keyNames.append(v)
#Parse extractedData
self.dataExtracted = []
for (k,v) in self.__paramfile.dataDict["extractedData"]:
assert k == "dataPoint"
assert isinstance(v,DataDict)
pb = {}
for (k,v) in v:
pb[k] = v
if not k in self.keyNames:
raise AcdOptiException_dataExtractor_loadFail("Key name '" + k + "' not found in self.keyNames")
self.dataExtracted.append(pb)
#Find, load, and attach filters
self.filters = []
for (k,v) in self.__paramfile.dataDict["filters"]:
self.filters.append(AcdOptiDataExtractorFilter.getFilterClass(k,v))
#Plots
self.plots = []
try:
filePlots = self.__paramfile.dataDict["plots"]
for (k,v) in filePlots:
self.plots.append(AcdOptiDataExtractorPlot.getPlotClass(k, self, v))
except AcdOptiException_dataDict_getValsSingle:
self.__paramfile.dataDict.pushBack("plots", DataDict())
self.__paramfile.write()
def runExtractor(self):
assert self.lockdown == False
#self.dataExtracted = []
#self.keyNames = []
#Reset filter counters
for f in self.filters:
f.numFiltered = 0
self.keyNames.append("META.GeomInstName")
self.keyNames.append("META.ScanInstNames")
self.keyNames.append("META.MeshInstName")
self.keyNames.append("META.rcInstName")
#Run Loop over all runconfigs and for each extract one row in the table
geomCollection = self.collection.project.geomCollection
for geom in geomCollection.geomInstances.itervalues():
geomData = {}
geomData["META.GeomInstName"] = geom.instName
if len(geom.scanInstances) > 0:
siString = ""
for si in geom.scanInstances:
siString += si.instName + " "
geomData["META.ScanInstNames"] = siString[:-1]
geomOverrideList = geom.templateOverrides_getKeys()
for key in geomCollection.paramDefaults_getKeys():
geomKey = "GEOM."+key
if key in geomOverrideList:
geomData[geomKey] = geom.templateOverrides_get(key)
else:
geomData[geomKey] = geomCollection.paramDefaults_get(key)
if not geomKey in self.keyNames:
self.keyNames.append(geomKey)
for mesh in geom.meshInsts.itervalues():
meshData = {}
meshData["META.MeshInstName"] = mesh.instName
meshOverrideList = mesh.templateOverrides_getKeys()
for key in mesh.meshTemplate.paramDefaults_getKeys():
meshKey = "MESH." + mesh.meshTemplate.instName + "." + key
if key in meshOverrideList:
meshData[meshKey] = mesh.templateOverrides_get(key)
else:
meshData[meshKey] = mesh.meshTemplate.paramDefaults_get(key)
if not meshKey in self.keyNames:
self.keyNames.append(meshKey)
for rc in mesh.runConfigs.itervalues():
pb = {}
pb["META.rcInstName"] = rc.instName
#Extract toward root: Geom and mesh data
pb.update(geomData)
pb.update(meshData)
#Extract data in RC
#TODO
#Extract from analysis
for (anaName,anaObj) in rc.analysis.iteritems():
#Recursively dig through keys in analysis
def anaDigger(exportedDict,nameSoFar):
keys = set(exportedDict.getKeys())
for k in keys:
vals = exportedDict.getVals(k)
for vIdx in xrange(len(vals)):
anaKey = nameSoFar + k + "["+str(vIdx)+"]"
if type(vals[vIdx]) == str:
pb[anaKey] = vals[vIdx]
if not anaKey in self.keyNames:
self.keyNames.append(anaKey)
else:
anaDigger(vals[vIdx],anaKey+".")
#END anaDigger
anaDigger(anaObj.exportResults,"ANA." + anaName + ".")
#Filter and store result
if reduce(lambda a,b: a and b, map(lambda f: f.filterRow(pb), self.filters) + [True]):
self.dataExtracted.append(pb)
#Sort keyName, first as as {META < GEOM < MESH < ANA}, then alphabetically after the first "." (but sort commands must be reversed)
def sort1(s):
if s.startswith("META."):
return 1
elif s.startswith("GEOM."):
return 2
elif s.startswith("MESH."):
return 3
elif s.startswith("ANA."):
return 4
else:
raise KeyError
self.keyNames.sort(key=lambda s: s.split(".",1)[1])
self.keyNames.sort(key=sort1)
#Filter keys
if len(self.keepKeys) > 0:
print "Filtering keys..."
i = 0
while i < len(self.keyNames):
k = self.keyNames[i]
if k in self.keepKeys:
print "Kept key '" + k + "'"
i += 1
else:
print "Deleting key '" + k + "'"
del self.keyNames[i]
for d in self.dataExtracted:
try:
del d[k]
except KeyError:
#print "key '" + k + "' not in this row"
pass
#i = 0 #restart loop
self.lockdown = True
self.write()
def export(self,fname=None,useKeys=None):
"""
Export the data as a CSV file suitable for import to R, spreadsheet etc.
The useKeys argument lets you specify which columns you want to use.
"""
if fname == None:
fname = self.extractFname
print "AcdOptiDataExtractor::export(), fname='" + fname + "'"
ofile = open(fname,"w")
if useKeys==None:
keys = self.keyNames
else:
keys = useKeys
#Write header
hline = ""
for k in keys:
hline += k + ", "
ofile.write(hline[:-2] + "\n")
#Data
for row in self.dataExtracted:
rline = ""
for k in keys:
try:
#ofile.write(row[k] + ", ")
rline += row[k] + ", "
except KeyError:
#ofile.write(" , ")
rline += ", "
#ofile.write("\n")
ofile.write(rline[:-2] + "\n")
ofile.close()
def clearLockdown(self):
self.dataExtracted = []
self.keyNames = []
self.lockdown = False
self.write()
def write(self):
self.__paramfile.dataDict.setValSingle("lockdown", str(self.lockdown))
self.__paramfile.dataDict.setValSingle("extractFname", self.extractFname)
knfile = self.__paramfile.dataDict["keyNames"]
knfile.clear()
for kn in self.keyNames:
knfile.pushBack("keyName", kn)
edfile = self.__paramfile.dataDict["extractedData"]
edfile.clear()
for row in self.dataExtracted:
rowDict = DataDict()
for (colKey,col) in row.iteritems():
rowDict.pushBack(colKey, col)
edfile.pushBack("dataPoint",rowDict)
fifile = self.__paramfile.dataDict["filters"]
fifile.clear()
for f in self.filters:
f.settingsDict["numFiltered"] = str(f.numFiltered)
fifile.pushBack(f.filterType, f.settingsDict)
kkfile = self.__paramfile.dataDict["keepKeys"]
kkfile.clear()
for k in self.keepKeys:
if len(self.keyNames) > 0 and not k in self.keyNames:
print "WARNING: Invalid key '" + k + "' in keepKeys"
kkfile.pushBack("key", k)
plfile = self.__paramfile.dataDict["plots"]
plfile.clear()
for p in self.plots:
plfile.pushBack(p.plotType, p.settingsDict)
self.__paramfile.write()
@staticmethod
def createNew(folder):
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
if os.path.isdir(folder):
raise AcdOptiException_dataExtractor_createFail("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create paramfile
paramFile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "AcdOptiDataExtractor")
paramFile.dataDict.pushBack("instName", instName)
paramFile.dataDict.pushBack("extractFname", "")
paramFile.dataDict.pushBack("keyNames", DataDict())
paramFile.dataDict.pushBack("extractedData", DataDict())
paramFile.dataDict.pushBack("filters", DataDict())
paramFile.dataDict.pushBack("plots", DataDict())
paramFile.dataDict.pushBack("keepKeys", DataDict())
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.write()
class AcdOptiRunner_Hopper(AcdOptiRunner):
type = "Hopper"
CPUsPerNode = 24
hostname = "hopper.nersc.gov"
#commonExecs = {"Omega3P::2011May23":"~candel/.community/hopper2/omega3p-2011May23"}
__sshClient = None #Try to keep using the same SSHClient
__paramFile = None
#PBSjobName = None
remoteJobID = None
speedSSH = None
def __init__(self,runConfig):
self.runConfig = runConfig
self.folder = self.runConfig.folder
self.__paramFile = AcdOptiFileParser_simple(os.path.join(self.folder,"paramFile_acdOptiRunner_Hopper.set"), 'rw')
if not self.__paramFile.dataDict["fileID"] == "AcdOptiRunner_Hopper":
raise AcdOptiException_optiRunner_loadFail("Wrong fileID, got'"+self.__paramFile.dataDict["fileID"] + "'")
self.remoteJobID = self.__paramFile.dataDict["remoteJobID"]
if self.remoteJobID == "":
self.remoteJobID = None
if self.remoteJobID != None and not self.runConfig.status.startswith("remote::"):
print "WARNING: Found remoteJobID, but status='" + self.runConfig.status + "'"
#raise AcdOptiException_optiRunner_loadFail("Found remoteJobID, but status='" + self.runConfig.status + "'")
elif self.remoteJobID == None and (self.runConfig.status == "remote::queued" or self.runConfig.status == "remote::running"):
raise AcdOptiException_optiRunner_loadFail("Did not find remoteJobID, but status='" + self.runConfig.status + "'")
self.speedSSH = SSHspeedtransfer(self.hostname, AcdOptiSettings().getSetting("hopperUser"))
def getTorqueMeta(self):
"Returns a pointer to the TorqueMeta data structure"
return self.__paramFile.dataDict["TorqueMeta"]
def getJobs(self):
"Return a pointer to the jobs data structure"
return self.__paramFile.dataDict["jobs"]
def isRemote(self):
return True
def __connectSSH(self):
"""
Method that setups a ssh connection, returning a paramiko.SSHClient object.
Please close() any SFTPClients opened, but don't close the client itself,
as this method tries to reuse an existing client (connecting is what takes most of the time)
"""
print "Connecting..."
if self.__sshClient != None:
print "Found an old client"
if self.__sshClient.get_transport() == None or self.__sshClient.get_transport().is_active() != True:
print "Old client not active."
else:
"Old client OK!"
return self.__sshClient
print "Couldn't use old client, creating a new one."
username = AcdOptiSettings().getSetting("hopperUser")
client = paramiko.SSHClient()
client.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))
client.connect(self.hostname, username=username)
#client.load_system_host_keys()
print "Connected."
self.__sshClient = client
return client
def __SSHkillDir(self, dir, sftp):
"""
Recursively delete directory dir and its contents using a sftp connection
"""
#Emptying the folder...
iDir = ""
if dir[-1] == "/":
iDir = dir[:-1]
else:
iDir = dir
fileList = sftp.listdir(iDir)
for file in fileList:
try:
sftp.remove(iDir + "/" + file)
except IOError:
#Directory
self.__SSHkillDir(iDir + "/" + file, sftp)
sftp.rmdir(iDir)
def upload(self):
stageFile = self.runConfig.stageFile
#Check for programming errors
assert stageFile != None
assert os.path.isfile(stageFile)
#Setup the ssh connection
username = AcdOptiSettings().getSetting("hopperUser")
client = self.__connectSSH()
#Look for the acdopti scratch directory
sftp = client.open_sftp()
remoteDir = "/scratch/scratchdirs/" + username + "/"
remoteScratch = remoteDir + "acdopti_scratch/"
remoteDirList = sftp.listdir(remoteDir)
if not "acdopti_scratch" in remoteDirList:
print "Making directory..."
print remoteDirList
sftp.mkdir(remoteScratch)
scratchDirList = sftp.listdir(remoteScratch)
#Put the file
if os.path.split(stageFile)[1] in scratchDirList:
print "File already on HPC?!?"
#client.close()
sftp.close()
return
print "Uploading file..."
remoteFile = remoteScratch + os.path.split(stageFile)[1]
self.speedSSH.put(stageFile, remoteScratch + os.path.split(stageFile)[1])
#sftp.put(stageFile, remoteScratch + os.path.split(stageFile)[1])
print "Uploading finished."
#Unzip
print "Unzipping..."
print "COMMAND:", "cd " + remoteScratch +"; tar xzvf " + remoteFile # + " --directory " + remoteScratch
(ssh_stdin, ssh_stdout, ssh_stderr) = client.exec_command("cd " + remoteScratch +"; tar xzvf " + remoteFile) #+ " --directory " + remoteScratch)
(ssh_stdout_str, ssh_stderr_str) = (ssh_stdout.read(), ssh_stderr.read())
print "STDOUT:", ssh_stdout_str
print "STDERR:", ssh_stderr_str
print "Unzipped."
if len(ssh_stderr_str):
#client.close()
sftp.close()
raise AcdOptiException_optiRunner_remoteProblem("Problem while unzipping, see output")
#Delete the remote tar.gz
print "Deleting tar.gz..."
dirList = sftp.listdir(remoteScratch)
if os.path.split(self.runConfig.stageFile)[1] in dirList:
sftp.remove(remoteFile)
print "Deleted."
else:
print "Already gone."
sftp.close()
#client.close()
def remoteCleanup(self):
assert self.runConfig.status in ["remote::uploaded", "remote::finished", "remote::unclean"]
#Make connection...
username = AcdOptiSettings().getSetting("hopperUser")
client = self.__connectSSH()
sftp = client.open_sftp()
remoteDir = "/scratch/scratchdirs/" + username + "/"
remoteScratch = remoteDir + "acdopti_scratch/"
remoteFile = remoteScratch + os.path.split(self.runConfig.stageFile)[1]
remoteFinishedFile = remoteScratch + self.runConfig.stageName + "--finished.tar.gz"
#Delete the remote tar.gz's
print "Deleting remote stage tar.gz..."
dirList = sftp.listdir(remoteScratch)
if os.path.split(self.runConfig.stageFile)[1] in dirList:
sftp.remove(remoteFile)
print "Deleted."
else:
print "Already gone."
print "Deleting remote finished tar.gz..."
if os.path.split(remoteFinishedFile)[1] in dirList:
sftp.remove(remoteFinishedFile)
print "Deleted."
else:
print "Already gone."
#Delete the remote folder
print "Deleting remote folder..."
if self.runConfig.stageName in dirList:
self.__SSHkillDir(remoteScratch + "/" + self.runConfig.stageName, sftp)
else:
print "Already gone."
sftp.close()
client.close()
def run(self):
#Make connection...
username = AcdOptiSettings().getSetting("hopperUser")
client = self.__connectSSH()
remoteDir = "/scratch/scratchdirs/" + username + "/"
remoteScratch = remoteDir + "acdopti_scratch/"
#Submit job
print "Submitting..."
(ssh_stdin, ssh_stdout, ssh_stderr) = client.exec_command("cd " + remoteScratch + self.runConfig.stageName + "; qsub " + remoteScratch + self.runConfig.stageName + "/run.pbs")
(ssh_stdout_str, ssh_stderr_str) = (ssh_stdout.read(), ssh_stderr.read())
print "STDOUT:", ssh_stdout_str
print "STDERR:", ssh_stderr_str
print "Submitted."
#client.close()
if len(ssh_stderr_str):
raise AcdOptiException_optiRunner_remoteProblem("Problem during submission, see output")
#Check if the stdout matches XXXXX.YYY, where XXXXX is a number, and YYY is letters.
# This is then the job ID.
if re.match("[0-9]+\.\w+$", ssh_stdout_str):
self.remoteJobID = ssh_stdout_str.strip()
self.__paramFile.dataDict.setValSingle("remoteJobID", self.remoteJobID)
print "Submission successful, JobID='" + self.remoteJobID + "'"
else:
raise AcdOptiException_optiRunner_runError("Problem with job submission, see standard output")
self.write()
def cancelRun(self):
assert self.remoteJobID != None
#Make connection...
client = self.__connectSSH()
#Cancel the current job
print "Issuing cancel command..."
(ssh_stdin, ssh_stdout, ssh_stderr) = client.exec_command("qdel " + self.remoteJobID)
(ssh_stdout_str, ssh_stderr_str) = (ssh_stdout.read(), ssh_stderr.read())
print "STDOUT:", ssh_stdout_str
print "STDERR:", ssh_stderr_str
print "Cancel command issued."
#client.close()
if len(ssh_stderr_str):
if "Unknown Job Id " + self.remoteJobID in ssh_stderr_str:
#Aready finished
print "Job was already finished"
self.remoteJobID = None
self.write()
return
elif "errno=15096" in ssh_stderr_str:
#Something went wrong with TORQUE on the server - but the job is very much dead...
print "Torque problem, but job is dead. (see output)"
self.remoteJobID = None
self.write()
return
raise AcdOptiException_optiRunner_remoteProblem("Problem during cancel, see output")
self.write()
def queryStatus(self):
assert self.runConfig.status == "remote::queued" or self.runConfig.status == "remote::running", "status = '" + self.runConfig.status + "'"
#Make connection
client = self.__connectSSH()
print "Getting status..."
(ssh_stdin, ssh_stdout, ssh_stderr) = client.exec_command("qstat " + self.remoteJobID)
(ssh_stdout_str, ssh_stderr_str) = (ssh_stdout.read(), ssh_stderr.read())
print "STDOUT:", ssh_stdout_str
print "STDERR:", ssh_stderr_str
print "Got status."
#client.close()
#Parse the status output:
if len(ssh_stderr_str):
if "Unknown Job Id Error " + self.remoteJobID in ssh_stderr_str:
self.remoteJobID = None
self.write()
return "remote::finished"
raise AcdOptiException_optiRunner_remoteProblem("Problem while getting status, see output")
statusline = ""
for line in ssh_stdout_str.splitlines():
if line.startswith(self.remoteJobID):
statusline = line
break
statusChar = statusline.split()[-2]
print "statusLine='" + statusline + "', statusChar='" + statusChar + "'"
if statusChar == "Q":
return "remote::queued"
elif statusChar == "R" or statusChar == "E" or statusChar == "T": #E: Exiting after having run, T: transfer
return "remote::running"
elif statusChar == "C":
self.remoteJobID = None
self.write()
return "remote::finished"
else:
raise ValueError("Unknown status char '" + statusChar + "'")
def getRemoteData(self):
assert self.runConfig.status=="remote::finished" or self.runConfig.status=="remote::unclean"
#assert self.remoteJobID == None
if self.remoteJobID != None and not self.runConfig.status.startswith("remote::"):
print "WARNING: Found remoteJobID, but status='" + self.runConfig.status + "'"
finishedLocalPath=os.path.join(self.folder, "finished")
username = AcdOptiSettings().getSetting("hopperUser")
remoteDir = "/scratch/scratchdirs/" + username + "/"
remoteScratch = remoteDir + "acdopti_scratch/"
#remoteJobDir = remoteScratch + self.runConfig.stageName + "/"
remoteFile = self.runConfig.stageName + "--finished.tar.gz"
#Make connection
client = self.__connectSSH()
#sftp = client.open_sftp()
#Tar the data
print "Zipping..."
command = "cd " + remoteScratch +"; tar czvf " + remoteFile + " --force-local " + self.runConfig.stageName
print "COMMAND:", command
(ssh_stdin, ssh_stdout, ssh_stderr) = client.exec_command(command)
(ssh_stdout_str, ssh_stderr_str) = (ssh_stdout.read(), ssh_stderr.read())
print "STDOUT:", ssh_stdout_str
print "STDERR:", ssh_stderr_str
print "Zipped."
if len(ssh_stderr_str):
#client.close()
raise AcdOptiException_optiRunner_remoteProblem("Problem during zipping, see output")
#Download the tarball
self.speedSSH.get(remoteScratch + remoteFile, os.path.join(finishedLocalPath, remoteFile))
#sftp.get(remoteScratch + remoteFile, os.path.join(finishedLocalPath, remoteFile)) #TOO SLOW over transatlantic link...
#sftp.close()
#client.close()
#Unzip the downloaded solution tar.gz
archive = tarfile.open(os.path.join(finishedLocalPath, remoteFile), "r:gz")
archive.extractall(path=finishedLocalPath)
return os.path.join(finishedLocalPath, self.runConfig.stageName) #Duplicated code in runConfig::init()!
def stage(self):
self.__makePBS()
def __makePBS(self):
"""
Creates the run.pbs file used by Hopper's batch system
"""
#Calculate number of mpi nodes needed, and build the commands
jobs = self.__paramFile.dataDict["jobs"]
commands = []
numNodes = 0
for jobName, job in jobs:
command = None
if DataDict.boolconv(job["aprun"]):
if job["tasksNode"] == "-1":
nodesThis = int(math.ceil(int(job["tasks"])/float(self.CPUsPerNode)))
else:
assert int(job["tasksNode"]) <= self.CPUsPerNode
nodesThis = int(math.ceil(int(job["tasks"])/float(job["tasksNode"])))
if nodesThis > numNodes:
numNodes = nodesThis
def makeOption(optionName, key, optional):
get = job[key]
if get == "-1" and optional:
return ""
return optionName + " " + get + " "
command = "aprun " + makeOption("-n", "tasks", False)\
+ makeOption("-N", "tasksNode", True)\
+ makeOption("-S", "tasksNuma", True)\
+ job["command"] + " " + job["commandArgs"]
else:
command = job["command"]
commands.append(command)
if len(commands) == 0:
raise AcdOptiException_optiRunner_stageError("No commands built")
if not numNodes > 0:
raise AcdOptiException_optiRunner_stageError("Got numNodes="+str(numNodes))
#Write PBS file header
runpbs = open(os.path.join(self.runConfig.stageFolder, "run.pbs"), 'w')
runpbs.write("#!/bin/bash\n")
torqueMeta = self.getTorqueMeta()
runpbs.write("#PBS -q " + torqueMeta["queue"] + "\n")
runpbs.write("#PBS -l mppwidth=" + str(numNodes*self.CPUsPerNode) + "\n")
runpbs.write("#PBS -l walltime=" + torqueMeta["walltime"] + "\n")
runpbs.write("#PBS -N " + self.runConfig.stageName + "\n")
runpbs.write("#PBS -A " + torqueMeta["repo"] + "\n")
if DataDict.boolconv(torqueMeta["importVars"]):
runpbs.write("#PBS -V\n")
runpbs.write("\n\n")
#Write PBS script
runpbs.write("## Commands:\n")
for command in commands:
runpbs.write(command + "\n")
runpbs.close()
#Move it to stage folder
#os.rename(os.path.join(self.folder, "run.pbs"), os.path.join(self.folder, "stage", "run.pbs"))
def __del__(self):
if self.__sshClient != None:
self.__sshClient.close()
def write(self):
self.__paramFile.write()
def cloneInto(self,cloneFrom):
"""
Empty this runner and copy the data from cloneFrom.
"""
#Copy the torque stuff
torqueOriginal = cloneFrom.getTorqueMeta()
torqueThis = self.getTorqueMeta()
torqueThis.clear()
for (k,v) in torqueOriginal.copy():
torqueThis.pushBack(k,v)
#Copy the jobs stuff
jobsOriginal = cloneFrom.getJobs()
jobsThis = self.getJobs()
jobsThis.clear()
for (k,v) in jobsOriginal.copy():
jobsThis.pushBack(k,v)
self.write()
@staticmethod
def createNew(folder, type):
#Create the settings file
paramFile = AcdOptiFileParser_simple(os.path.join(folder,"paramFile_acdOptiRunner_Hopper.set"),"w")
paramFile.dataDict.pushBack("fileID", "AcdOptiRunner_Hopper")
paramFile.dataDict.pushBack("remoteJobID", "")
#Set default torque meta stuff
torqueMeta = paramFile.dataDict.pushBack("TorqueMeta", DataDict())
torqueMeta.pushBack("queue", "regular")
torqueMeta.pushBack("walltime", "00:59:00")
torqueMeta.pushBack("repo", "m349")
torqueMeta.pushBack("importVars", "True")
#Create a datastructure for storing aprun jobs
jobs = paramFile.dataDict.pushBack("jobs", DataDict())
# Each aprun job has the following fields:
# - aprun: Boolean, true in the case of aprun jobs
# - command: Command to run
# - commandArgs: Arguments to pass to the executable (such as name of input file)
# - tasks: Number of MPI tasks, -n. Essential!
# - tasksNode: Number of MPI tasks pr. node, -N. Optional.
# - tasksNuma: Number of MPI tasks pr. NUMA node, -S. Optional.
# If the aprun flag is False, then only command is used (but all keys should be present!)
# Optional args should also be present. Set to "-1" to disable.
#This command is always needed.
cdpbs = jobs.pushBack("cdPBS", DataDict())
cdpbs.pushBack("aprun", "False")
cdpbs.pushBack("command", "cd $PBS_O_WORKDIR")
cdpbs.pushBack("commandArgs", "")
cdpbs.pushBack("tasks", "-1")
cdpbs.pushBack("tasksNode", "-1")
cdpbs.pushBack("tasksNuma", "-1")
paramFile.write()
def __init__(self, folder, collection):
self.folder = folder
self.collection = collection
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
self.instName = instName = os.path.split(instName)[1]
#Load paramFile
self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw')
if self.__paramfile.dataDict["fileID"] != "AcdOptiDataExtractor":
raise AcdOptiException_dataExtractor_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'")
if self.__paramfile.dataDict.getValSingle("instName") != self.instName:
raise AcdOptiException_dataExtractor_loadFail("instName doesn't match folder name")
self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown"))
try:
self.extractFname = self.__paramfile.dataDict["extractFname"]
except AcdOptiException_dataDict_getValsSingle:
print "AcdOptiDataExtractor::__init__(): Adding extractFname to paramFile"
self.__paramfile.dataDict.pushBack("extractFname", "")
self.__paramfile.write()
self.extractFname = ""
#KeepKeys
self.keepKeys = []
try:
fileKeepKeys = self.__paramfile.dataDict["keepKeys"]
for (k,v) in fileKeepKeys:
assert k == "key"
self.keepKeys.append(v)
except AcdOptiException_dataDict_getValsSingle:
print "AcdOptiDataExtractor::__init__(): Adding keepKeys to paramFile"
self.__paramfile.dataDict.pushBack("keepKeys", DataDict())
self.__paramfile.write()
#Parse keyNames
self.keyNames = []
for (k,v) in self.__paramfile.dataDict["keyNames"]:
assert k == "keyName"
self.keyNames.append(v)
#Parse extractedData
self.dataExtracted = []
for (k,v) in self.__paramfile.dataDict["extractedData"]:
assert k == "dataPoint"
assert isinstance(v,DataDict)
pb = {}
for (k,v) in v:
pb[k] = v
if not k in self.keyNames:
raise AcdOptiException_dataExtractor_loadFail("Key name '" + k + "' not found in self.keyNames")
self.dataExtracted.append(pb)
#Find, load, and attach filters
self.filters = []
for (k,v) in self.__paramfile.dataDict["filters"]:
self.filters.append(AcdOptiDataExtractorFilter.getFilterClass(k,v))
#Plots
self.plots = []
try:
filePlots = self.__paramfile.dataDict["plots"]
for (k,v) in filePlots:
self.plots.append(AcdOptiDataExtractorPlot.getPlotClass(k, self, v))
except AcdOptiException_dataDict_getValsSingle:
self.__paramfile.dataDict.pushBack("plots", DataDict())
self.__paramfile.write()
class AcdOptiSolverManager:
"""
This class does the dirtywork of AcdOptiSolverSetup,
such as creating the actual solver setup files.
It generates a solver setup from the specified type.
If the solution name exists, it will be loaded,
else it will be created from default values.
AcdOptiSolverSetup remains as an alternative interface to this class,
with simpler initialization given by an AcdOptiRunConfig
"""
# Constants
# Data types
fileTypes = ["simple", "Lua", "KVC"]
dataTypes = ["dict", "fnameNoEx", "fnameEx", "string", "bool", "int", "intList", "float"]
# dict: Dictionary. Have key/DataDict "children" with sub-keys
# fnameNoEx : relative path of file that does not exist
# fnameEx : relative path of file that does exist
# string : a string without trailing/leading whitespace
# bool : a bool, coded as "True"/"False" or "true"/"false". If key "int" is True, coded as "1"/"0"
# int : an integer
# intList : a list of integers "a,b,c"
# float : a float
# Object fields
type = None # Type of the setup file
# (omega3P etc.; name of one of the .set files found in solverSetupResourcesPath)
name = None # Name of the generated setup file
folder = None # Folder where this file is located
fileName = None # Full path of the setup file
__metaSetupFile = None # AcdOptiFileParser_simple object with metadata and current settings
metaSetupFilePath = None # Full path to the MetaSetupFile
metaSetup = None # DataDict object pointing to metaSetupFile.dataDict["options"] (shortcut pointer)
setupFileFormat = None # Pointer to correct AcdOptiFileParser_* class
lockdown = False # Is the solverSetup currently not writable? (not enforced, but influences GUI)
def __init__(self, name, folder):
"""
Initialize a solverManager.
- Name: Name of this instance (used to find the metaFile and name of the created setup file)
- folder: Where the metaFile lives, and where the setup file is created.
"""
print "AcdOptiSolverManager::__init__(), name='" + name + "', folder='" + folder + "'"
self.folder = folder
self.name = name
if not os.path.isdir(self.folder):
raise AcdOptiException_solverSetup_loadFail("Subfolder 'stage' is missing")
# Load the metaFile
self.metaSetupFilePath = metaFileName = os.path.join(self.folder, name + ".meta")
self.__metaSetupFile = AcdOptiFileParser_simple(metaFileName, "rw")
if self.__metaSetupFile.dataDict["fileID"] != "SolverManager":
if self.__metaSetupFile.dataDict["fileID"] == "SolverSetup":
print "\t WARNING: Old-style file detected, fileID='SolverSetup'"
print "\t Will change it to new-style 'solverManager'"
self.__metaSetupFile.dataDict.setValSingle("fileID", "SolverManager")
self.__metaSetupFile.write()
else:
raise AcdOptiException_solverSetup_loadFail(
"Wrong fileID in metaSetupFile '" + self.metaSetupFilePath + "'"
)
if self.name != self.__metaSetupFile.dataDict["name"]:
raise AcdOptiException_solverSetup_loadFail(
"name in metaSetupFile '"
+ self.__metaSetupFile.dataDict["name"]
+ "' doesn't match provided argument '"
+ self.name
+ "'"
)
self.setupFileFormat = self.__metaSetupFile.dataDict.getValSingle("fileFormat")
if self.setupFileFormat == "simple":
self.setupFileFormat = AcdOptiFileParser_simple
elif self.setupFileFormat == "Lua":
self.setupFileFormat = AcdOptiFileParser_Lua
elif self.setupFileFormat == "KVC":
self.setupFileFormat = AcdOptiFileParser_KVC
else:
raise ValueError("Unknown setupFileFormat encountered, setupFileFormat='" + self.setupFileFormat + "'")
self.type = self.__metaSetupFile.dataDict["type"]
self.metaSetup = self.__metaSetupFile.dataDict.getValSingle("options")
self.fileName = os.path.join(self.folder, self.name)
def generateSetup(self):
"Generates a setupFile from the current contents of metaSetup, which is written in the given folder."
print "AcdOptiSolverSetupManager::__generateSetup()"
self.write()
setupFile = self.setupFileFormat(self.fileName, "w")
metaSetupDict = AcdOptiSolverManager.__generateSetup_recursiveHelper(self.metaSetup)
setupFile.importDataDict(metaSetupDict)
setupFile.write()
@staticmethod
def __generateSetup_recursiveHelper(setupDict):
print "AcdOptiSolverManager::__generateSetup_recursiveHelper()" # , setupDict=", setupDict
ret = DataDict()
for item in setupDict:
if not DataDict.boolconv(item[1]["enabled"]):
continue
if item[1]["type"] == "dict":
ret.pushBack(item[0], AcdOptiSolverManager.__generateSetup_recursiveHelper(item[1]["children"]))
else:
ret.pushBack(item[0], item[1]["value"])
return ret
def write(self):
self.__metaSetupFile.write()
@staticmethod
def getTypes():
"""
Returns a report listing all valid 'types'
"""
fileList = os.listdir(solverSetupResourcesPath)
typeList = []
for file in fileList:
if os.path.isfile(os.path.join(solverSetupResourcesPath, file)) and file.endswith(".set"):
typeList.append(file[:-4])
return typeList
@staticmethod
def createNew(type, folder, name=None):
"""
Prepares a new AcdOptiSolverManager by creating the metaFile.
Input:
- type is the name of the type ("omega3P" etc.) wanted
- folder is where this instance should be created
- name is the name wanted for this instance.
Set to None to use default name.
Name is also the filename of the solver setup file generated.
Returns the name used.
"""
print "AcdOptiSolverSetupManager::createNew(), type=" + type + ", name=" + str(name)
# Check that the type is valid
if not type in AcdOptiSolverManager.getTypes():
raise TypeError("type '" + type + "' is invalid")
# Load the type definition
typeFile = AcdOptiFileParser_simple(os.path.join(solverSetupResourcesPath, type + ".set"), "r")
if typeFile.dataDict["fileID"] != "SolverSetupTemplate":
raise AcdOptiException_solverSetup_createFail("Got fileID ='" + str(typeFile.dataDict["fileID"]) + "'")
# Check that everything is OK with the typeFile
# TODO!
# Read the typeFile
if not name:
name = typeFile.dataDict.getValSingle("fileNameDefault")
# Generate the metaFile
metaFileName = os.path.join(folder, name + ".meta")
# First: check that the name is not in use
if os.path.isfile(os.path.join(folder, metaFileName)):
raise AcdOptiException_solverSetup_createFail_nameTaken(
"Name already in use, couldn't create metaFile", name
)
metaFile = AcdOptiFileParser_simple(os.path.join(folder, metaFileName), "w")
# Setup the basic structure of the metaFile
metaFile.dataDict.pushBack("fileID", "SolverManager")
metaFile.dataDict.pushBack("name", name)
metaFile.dataDict.pushBack("fileFormat", typeFile.dataDict["fileFormat"])
metaFile.dataDict.pushBack("type", typeFile.dataDict["type"])
# Generate the options (which in itself is a childDict)
metaFile.dataDict.pushBack(
"options", AcdOptiSolverManager.__genMetaOptions(typeFile.dataDict.getValSingle("options"))
)
metaFile.write()
return name
@staticmethod
def createNew_clone(folder, cloneFrom):
"""
Creates a new solverSetup in a not previously existing folder,
which has identical settings as an already existing solverSetup.
The newly created solverSetup is then returned.
"""
AcdOptiSolverManager.createNew(cloneFrom.type, folder, cloneFrom.name)
newSolverSetup = AcdOptiSolverManager(cloneFrom.name, folder)
newSolverSetup.metaSetup.clear()
for (k, v) in cloneFrom.metaSetup.copy():
newSolverSetup.metaSetup.pushBack(k, v)
newSolverSetup.write()
return newSolverSetup
@staticmethod
def __genMetaOptions(childDict):
"""
Recursive method used by createNew() for generating the "options" part of the metaFile.
Returns a DataDict which becomes one branch/level of the metaFile
"""
print "AcdOptiSolverSetupManager::__genMetaOptions()"
ret = DataDict()
for item in childDict:
# Make a copy of the old items in the childDict and add it to the metFile
thisItem = ret.pushBack(item[0], item[1].copy())
# Add some new meta-information fields
if thisItem.getValSingle("type") == "dict":
# Recursively modify this dicts children also
thisItem.setValSingle(
"children", AcdOptiSolverManager.__genMetaOptions(item[1].getValSingle("children"))
)
else:
# Ordinary data field, set its value to whatever is default
thisItem.pushBack("value", thisItem.getValSingle("default"))
if DataDict.boolconv(item[1].getValSingle("must")):
thisItem.pushBack("enabled", "True")
else:
thisItem.pushBack("enabled", "False")
return ret
@staticmethod
def isInputInvalid(itemDict, newValue=None):
"""
Check whether newValue is a valid value of the itemDict
by the rules encoded in the itemDict.
If newValue=None, check the current value instead.
If it is invalid, return an error message as a string,
else return None.
"""
if not itemDict["type"] in AcdOptiSolverManager.dataTypes:
return "Invalid type '" + itemDict["type"] + "'"
# TODO: implement some real checking for each type!
if itemDict["type"] == "dict":
return
else:
return
def __init__(self, folder, geometryInstance, meshTemplateCollection):
"""
Loads the meshInstance from the data
from folder and connects it to its template and geometryInstance.
Raises AcdOptiException_meshInstance_loadFail
if something (such as "not a meshInstance folder")
goes wrong.
"""
print "AcdOptiMeshInstance::__init__()"
self.folder = folder
self.geometryInstance = geometryInstance
self.meshTemplateCollection = meshTemplateCollection
if not os.path.isdir(folder):
raise AcdOptiException_meshInstance_loadFail(\
"Folder \"" + folder + "\" does not exist")
#Construct the instance name from folder name
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
instname = os.path.split(instname)[1]
self.instName = instname
#Load paramFile.set
try:
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'rw')
except IOError:
raise AcdOptiException_meshInstance_loadFail(\
"Problem loading file \"paramFile.set\" in folder \"" +\
folder + "\", got an IOError")
if self.__paramFile.dataDict.getValSingle("fileID") != "meshInstance":
raise AcdOptiException_meshInstance_loadFail(
"Wrong fileID in meshInstance.set, got \"" +
self.__paramFile.dataDict.getValSingle("fileID") + "\"")
lock = self.__paramFile.dataDict.getValSingle("lockdown")
if lock == "True":
self.lockdown = True
elif lock == "False":
self.lockdown = False
else:
raise AcdOptiException_meshInstance_loadFail(\
"Invalid content in field \"lockdown\" of file paramFile.set")
meshBad = self.__paramFile.dataDict.getVals("meshBad")
if len(meshBad) == 0:
self.meshBad = False
else:
assert len(meshBad) == 1
self.meshBad = DataDict.boolconv(meshBad[0])
#Check that the geometryInstance is correct
geometryInstance_expectedName = self.__paramFile.dataDict.getValSingle("geomInstance_name")
if geometryInstance.instName != geometryInstance_expectedName:
raise AcdOptiException_meshInstance_loadFail(\
"Excpected name of geometryInstance does not match the one passed")
#Find the MeshTemplate
meshTemplate_expectedName = self.__paramFile.dataDict.getValSingle("meshTemplate_name")
try:
self.meshTemplate = self.meshTemplateCollection.meshTemplates[meshTemplate_expectedName]
self.meshTemplate.registerInstance(self)
except KeyError:
raise AcdOptiException_meshInstance_loadFail("Could not find the meshTemplate \""
+ meshTemplate_expectedName + "\"")
#Find and load template overrides
self.__templateOverrides = {}
try:
templateOverrides_data = self.__paramFile.dataDict.getValSingle("templateOverrides")
except AcdOptiException_dataDict_getValsSingle:
raise AcdOptiException_meshInstance_loadFail\
("Couldn't load templateOverrides from paramFile.set")
if not isinstance(templateOverrides_data,DataDict):
raise AcdOptiException_meshInstance_loadFail\
("templateOverrides from paramFile.set is not a DataDict!")
for (k,v) in zip(templateOverrides_data.keys, templateOverrides_data.vals):
if k in self.__templateOverrides:
raise AcdOptiException_geomCollection_loadFail\
("Double occurrence of key \"" + k + "\" in templateOverrides on file")
if not k in self.meshTemplate.paramDefaults_getKeys():
raise AcdOptiException_meshInstance_loadFail(
"Entry \"" + k + "\" in templateOverrides on file has no match in the template")
self.__templateOverrides[k] = v
#Find subfolders and check if they are runConfigs
if not os.path.isdir(os.path.join(self.folder, "runConfigs")):
raise AcdOptiException_meshInstance_loadFail("Could not find runConfigs folder")
self.runConfigs = {}
for d in os.listdir(os.path.join(self.folder,"runConfigs")):
dAbs = os.path.abspath(os.path.join(self.folder,"runConfigs",d))
if not os.path.isdir(dAbs):
#Skip files etc.
continue
#try:
self.runConfigs[d] = AcdOptiRunConfig(dAbs,self)
class AcdOptiMeshInstance:
"""
This class is "owned" by a GeometryInstance,
where it represents the application of one MeshTemplate
onto the GeometryInstance.
It will handle variable overrides and call CUBIT in the same manner
as the GeometryInstance, and it will deliver a mesh in .netcdf format.
It is also the point of attachment for RunConfigs.
"""
#Object variables
geometryInstance = None
meshTemplateCollection = None
meshTemplate = None
runConfigs = None #RunConfig used on this MeshInstance
__paramFile = None
__templateOverrides = None
folder = None
instName = None
lockdown = False
meshBad = False #True if a mesh is generated and has ISOTEs
cubitMeshPreCommands = ["reset", "open 'geom.cub'"]
cubitMeshPostCommands = ["export genesis 'mesh.gen' block all overwrite"]
def __init__(self, folder, geometryInstance, meshTemplateCollection):
"""
Loads the meshInstance from the data
from folder and connects it to its template and geometryInstance.
Raises AcdOptiException_meshInstance_loadFail
if something (such as "not a meshInstance folder")
goes wrong.
"""
print "AcdOptiMeshInstance::__init__()"
self.folder = folder
self.geometryInstance = geometryInstance
self.meshTemplateCollection = meshTemplateCollection
if not os.path.isdir(folder):
raise AcdOptiException_meshInstance_loadFail(\
"Folder \"" + folder + "\" does not exist")
#Construct the instance name from folder name
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
instname = os.path.split(instname)[1]
self.instName = instname
#Load paramFile.set
try:
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'rw')
except IOError:
raise AcdOptiException_meshInstance_loadFail(\
"Problem loading file \"paramFile.set\" in folder \"" +\
folder + "\", got an IOError")
if self.__paramFile.dataDict.getValSingle("fileID") != "meshInstance":
raise AcdOptiException_meshInstance_loadFail(
"Wrong fileID in meshInstance.set, got \"" +
self.__paramFile.dataDict.getValSingle("fileID") + "\"")
lock = self.__paramFile.dataDict.getValSingle("lockdown")
if lock == "True":
self.lockdown = True
elif lock == "False":
self.lockdown = False
else:
raise AcdOptiException_meshInstance_loadFail(\
"Invalid content in field \"lockdown\" of file paramFile.set")
meshBad = self.__paramFile.dataDict.getVals("meshBad")
if len(meshBad) == 0:
self.meshBad = False
else:
assert len(meshBad) == 1
self.meshBad = DataDict.boolconv(meshBad[0])
#Check that the geometryInstance is correct
geometryInstance_expectedName = self.__paramFile.dataDict.getValSingle("geomInstance_name")
if geometryInstance.instName != geometryInstance_expectedName:
raise AcdOptiException_meshInstance_loadFail(\
"Excpected name of geometryInstance does not match the one passed")
#Find the MeshTemplate
meshTemplate_expectedName = self.__paramFile.dataDict.getValSingle("meshTemplate_name")
try:
self.meshTemplate = self.meshTemplateCollection.meshTemplates[meshTemplate_expectedName]
self.meshTemplate.registerInstance(self)
except KeyError:
raise AcdOptiException_meshInstance_loadFail("Could not find the meshTemplate \""
+ meshTemplate_expectedName + "\"")
#Find and load template overrides
self.__templateOverrides = {}
try:
templateOverrides_data = self.__paramFile.dataDict.getValSingle("templateOverrides")
except AcdOptiException_dataDict_getValsSingle:
raise AcdOptiException_meshInstance_loadFail\
("Couldn't load templateOverrides from paramFile.set")
if not isinstance(templateOverrides_data,DataDict):
raise AcdOptiException_meshInstance_loadFail\
("templateOverrides from paramFile.set is not a DataDict!")
for (k,v) in zip(templateOverrides_data.keys, templateOverrides_data.vals):
if k in self.__templateOverrides:
raise AcdOptiException_geomCollection_loadFail\
("Double occurrence of key \"" + k + "\" in templateOverrides on file")
if not k in self.meshTemplate.paramDefaults_getKeys():
raise AcdOptiException_meshInstance_loadFail(
"Entry \"" + k + "\" in templateOverrides on file has no match in the template")
self.__templateOverrides[k] = v
#Find subfolders and check if they are runConfigs
if not os.path.isdir(os.path.join(self.folder, "runConfigs")):
raise AcdOptiException_meshInstance_loadFail("Could not find runConfigs folder")
self.runConfigs = {}
for d in os.listdir(os.path.join(self.folder,"runConfigs")):
dAbs = os.path.abspath(os.path.join(self.folder,"runConfigs",d))
if not os.path.isdir(dAbs):
#Skip files etc.
continue
#try:
self.runConfigs[d] = AcdOptiRunConfig(dAbs,self)
#except AcdOptiException_runConfig_loadFail as e:
# raise AcdOptiException_meshInstance_loadFail(\
# "Problem loading assumed runConfig folder \"" + d + "\"", e)
def generatePattern(self):
"""
Generate substitution dict
"""
params = self.meshTemplate.paramDefaults_copy()
for k in self.__templateOverrides:
assert k in params
params[k] = self.__templateOverrides[k]
return params
def generateCubitJou(self):
"""
Uses the currently defined variables (overrides + inherited)
to generate a cubit script. Returns a tuple with
the generated script as a string, and a list of tag's not found in the template.
"""
params = self.generatePattern()
(runScript, notFound) = self.meshTemplate.meshTemplateFile.generateJou(params, self.cubitMeshPreCommands, self.cubitMeshPostCommands)
ret = ""
for line in runScript:
ret += line + "\n"
return (ret, notFound)
def generateMesh(self):
"""
Generates the CUBIT journal,
and runs this with the pre- and post-commands that are useful
for generating a mesh.
Also saves the journal as mesh.jou for later reference
Side-effect: Sets lockdown, and generates a solid if necessary
Returns any keys not found.
"""
print "AcdOptiMeshInstance::generateMesh()"
if not self.geometryInstance.lockdown:
self.geometryInstance.generateGeometry()
#Preparation
params = self.generatePattern()
templateFile = self.meshTemplate.meshTemplateFile
#Make the journal for later reference
refJouFileName = os.path.join(self.folder, "mesh.jou")
if os.path.isfile(refJouFileName):
os.remove(refJouFileName)
templateFile.writeJouFile(params,refJouFileName,self.cubitMeshPreCommands, self.cubitMeshPostCommands)
#Make the mesh
shutil.copy(os.path.join(self.geometryInstance.folder, "geom.cub"), os.getcwd())
notFound = templateFile.runCubit(params, preCommands=self.cubitMeshPreCommands,\
postCommands=self.cubitMeshPostCommands)
os.rename("mesh.gen", os.path.join(self.folder, "mesh.gen"))
#Convert the mesh to NetCDF
(acdoutput, badelems) = AcdOptiAcdtoolWrapper.convertGenNcdf(os.path.join(self.folder, "mesh.gen"),
os.path.join(self.folder, "mesh.ncdf"))
self.setLockdown()
if badelems > 0:
self.meshBad = True
self.write()
raise AcdOptiException_meshInstance_generateFail("Mesh had ISOTEs -- not a good mesh! (mesh still generated)")
else:
self.meshBad = False
self.write()
return notFound
def addRunConfig(self,name,runnerType, solverTypes=None):
"""
Creates and adds a new RunConfig with the specified
name, runnerType and solverTypes to this MeshInstance
"""
#Check that name is not already in use
if name in self.runConfigs:
raise AcdOptiException_meshInstance_nameError("RunConfig name already in use")
#Create the runConfig
runConfigFolder = os.path.join(self.folder, "runConfigs", name)
AcdOptiRunConfig.createNew(runConfigFolder, runnerType, solverTypes)
#Load it
self.runConfigs[name] = AcdOptiRunConfig(runConfigFolder, self)
def write(self):
"""
Update the contents of self.__paramFile to reflect
current status of the object
"""
print "AcdOptiMeshInstance::write()"
self.__paramFile.dataDict.setValSingle("lockdown", str(self.lockdown))
templateOverrides_data = self.__paramFile.dataDict.getValSingle("templateOverrides")
templateOverrides_data.clear()
for k in self.__templateOverrides:
templateOverrides_data.pushBack(k,self.__templateOverrides[k])
if len(self.__paramFile.dataDict.getVals("meshBad")) > 0:
self.__paramFile.dataDict.setValSingle("meshBad", str(self.meshBad))
else:
self.__paramFile.dataDict.pushBack("meshBad", str(self.meshBad))
self.__paramFile.write()
def setLockdown(self):
"""
Set lockdown = True,
indicating that a mesh has been generated, and that
the templateOverrides should not be touched.
Also sets lockdown on the geometryInstance and meshTemplate,
and writes itself to file.
"""
print "AcdOptiMeshInstance::setLockdown()"
self.geometryInstance.setLockdown()
self.meshTemplate.setLockdown()
self.lockdown = True
self.write()
def clearLockdown(self):
"""
Clears lockdown of the mesh instance,
and deletes generated cubit objects.
Also clears the lockdown on any RunConfig(s)
Also writes instance to file.
"""
print "AcdOptiGeometryInstance::clearLockdown()"
if not self.lockdown:
return
#Delete mesh files
if os.path.isfile(os.path.join(self.folder, "mesh.gen")):
os.remove(os.path.join(self.folder, "mesh.gen"))
if os.path.isfile(os.path.join(self.folder, "mesh.ncdf")):
os.remove(os.path.join(self.folder, "mesh.ncdf"))
if os.path.isfile(os.path.join(self.folder, "mesh.jou")):
os.remove(os.path.join(self.folder, "mesh.jou"))
#Delete vtu files from acdtool meshconvert
list = os.listdir(self.folder)
for l in list:
if l.endswith(".vtu"):
os.remove(os.path.join(self.folder, l))
for rc in self.runConfigs.values():
rc.clearLockdown()
self.meshBad = False
self.lockdown = False
self.write()
def templateOverrides_insert(self,key,val):
if self.lockdown:
raise AcdOptiException_meshInstance_lockdownError
self.__templateOverrides[key] = val
return val
def templateOverrides_get(self,key):
return self.__templateOverrides[key]
def templateOverrides_getKeys(self):
return self.__templateOverrides.keys()
def templateOverrides_del(self,key):
if self.lockdown:
raise AcdOptiException_meshInstance_lockdownError
val = self.__templateOverrides[key]
del self.__templateOverrides[key]
return val
def templateOverrides_clear(self):
if self.lockdown:
raise AcdOptiException_meshInstance_lockdownError
self.__templateOverrides.clear()
@staticmethod
def createNew(folder, geomInstance_name, meshTemplate_name):
"""
Creates a new empty meshInstance in a
not previously existing folder.
Folder name should be the same as meshInstance name.
Raises AcdOptiException_meshInstance_createFail
is something goes wrong (such as "Folder already exists")
"""
#Construct the instance name from folder
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
instname = os.path.split(instname)[1]
#Create the folder
if os.path.isdir(folder):
raise AcdOptiException_meshInstance_createFail ("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create paramFile.set file
paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "meshInstance")
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.dataDict.pushBack("templateOverrides", DataDict())
paramFile.dataDict.pushBack("meshTemplate_name", meshTemplate_name)
paramFile.dataDict.pushBack("geomInstance_name", geomInstance_name)
paramFile.write()
#Create folder for the runConfigs
os.mkdir(os.path.join(folder, "runConfigs"))
@staticmethod
def createNew_clone(folder, cloneFrom, newGeomInstance):
"""
Creates a new meshInstance in a not previously existing folder,
which has identical settings as an already existing meshInstance,
but is attached to newGeomInstance
The newly created meshInstance is then returned.
This is a deep copy, runConfigs etc. are also cloned.
"""
#Create the new meshInstance
AcdOptiMeshInstance.createNew(folder, newGeomInstance.instName, cloneFrom.meshTemplate.instName)
newInstance = AcdOptiMeshInstance(folder, newGeomInstance, cloneFrom.meshTemplateCollection)
#Copy information
for key in cloneFrom.templateOverrides_getKeys():
newInstance.templateOverrides_insert(key, cloneFrom.templateOverrides_get(key))
for (runConfigName, runConfig) in cloneFrom.runConfigs.iteritems():
newRC = AcdOptiRunConfig.createNew_clone(os.path.join(folder, "runConfigs", runConfig.instName), runConfig, newInstance)
newInstance.runConfigs[runConfigName] = newRC
newInstance.write()
return newInstance
def __init__(self, folder, scanCollection):
self.folder = folder
self.scanCollection = scanCollection
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
self.instName = instName = os.path.split(instName)[1]
#Load paramFile
self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw')
if self.__paramfile.dataDict["fileID"] != "AcdOptiScan":
raise AcdOptiException_scan_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'")
if self.__paramfile.dataDict.getValSingle("instName") != self.instName:
raise AcdOptiException_scan_loadFail("instName doesn't match folder name")
self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown"))
self.staged = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("staged"))
self.run = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("run"))
self.slaveGeomsDict = self.__paramfile.dataDict["slaveGeoms"]
self.slaveGeoms = []
baseGeomInstance_name = self.__paramfile.dataDict["baseGeomInstance_name"]
if baseGeomInstance_name == "":
self.baseGeomInstance = None
assert len(self.slaveGeomsDict) == 0
else:
self.baseGeomInstance = self.scanCollection.project.geomCollection.geomInstances[baseGeomInstance_name]
if self.__paramfile.dataDict["scanParameter_name"] != "":
#assert self.baseGeomInstance != None #Why?
self.scanParameter_name = self.__paramfile.dataDict["scanParameter_name"]
assert self.scanParameter_name in self.getValidParamNames()
self.scanParameter_range_max = float(self.__paramfile.dataDict["scanParameter_max"])
self.scanParameter_range_min = float(self.__paramfile.dataDict["scanParameter_min"])
self.scanParameter_range_step = float(self.__paramfile.dataDict["scanParameter_step"])
self.generateRange()
try:
self.predict_anaVariable = self.__paramfile.dataDict["predict_anaVariable"]
self.predict_targetValue = self.__paramfile.dataDict["predict_targetValue"]
except AcdOptiException_dataDict_getValsSingle:
self.predict_anaVariable = ""
self.predict_targetValue = ""
self.__paramfile.dataDict.pushBack("predict_anaVariable", "")
self.__paramfile.dataDict.pushBack("predict_targetValue", "")
self.__paramfile.write()
try:
self.predict_a = self.__paramfile.dataDict["predict_a"]
self.predict_b = self.__paramfile.dataDict["predict_b"]
self.predict_x = self.__paramfile.dataDict["predict_x"]
self.predict_r = self.__paramfile.dataDict["predict_r"]
except AcdOptiException_dataDict_getValsSingle:
self.predict_a = ""
self.predict_b = ""
self.predict_x = ""
self.predict_r = ""
self.__paramfile.dataDict.pushBack("predict_a", "")
self.__paramfile.dataDict.pushBack("predict_b", "")
self.__paramfile.dataDict.pushBack("predict_x", "")
self.__paramfile.dataDict.pushBack("predict_r", "")
self.__paramfile.write()
try:
self.predict_ndof=self.__paramfile.dataDict["predict_ndof"]
except AcdOptiException_dataDict_getValsSingle:
self.predict_ndof = ""
self.__paramfile.dataDict.pushBack("predict_ndof","")
self.__paramfile.write()
for (geomName, nothingOfInterest) in self.slaveGeomsDict:
#Mutal referencing
self.slaveGeoms.append(self.scanCollection.project.geomCollection.geomInstances[geomName])
self.slaveGeoms[-1].scanInstances.append(self)
def createNew(folder, runnerType, solverTypes=None):
"""
Create a new RunConfig, if wanted with ready-made
SolverSetups and Runners created and attached.
Input:
- Folder : In which folder to store the setup files etc.
- runnerType : String with the runner type wanted (example: Hopper)
- solverTypes : String or list of strings with the types of SolverSetups wanted
(as accepted by AcdOptiSolverSetup.createNew()).
If None, don't attach any solvers
"""
print "AcdOptiRunConfig::createNew(), folder=" + folder
#Construct the instance name from folder
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
instname = os.path.split(instname)[1]
if os.path.isdir(folder):
raise AcdOptiException_runConfig_createFail ("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create the SolverSetups:
solverSetups = []
if solverTypes == None:
pass #To simplify the "else" at the end...
elif type(solverTypes) == str:
solverSetups.append(AcdOptiSolverSetup.createNew(solverTypes, folder))
elif type(solverTypes) == list:
for st in solverTypes:
solverSetups.append(AcdOptiSolverSetup.createNew(st, folder))
else:
raise AcdOptiException_runConfig_createFail ("Expected solverTypes to be a string or list of strings, got " + str(solverTypes))
#Create the runner
AcdOptiRunner.createNew(runnerType, folder)
#Create the paramFile
paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "runConfigParamFile")
paramFile.dataDict.pushBack("instName", instname)
if len(solverSetups):
paramFile.dataDict.pushBack("status", "initialized")
else:
paramFile.dataDict.pushBack("status", "not_initialized")
paramFile.dataDict.pushBack("stageName", "")
# paramFile.dataDict.pushBack("stageFolder", "")
# paramFile.dataDict.pushBack("stageFile", "")
# paramFile.dataDict.pushBack("finishedFolder", "")
#Pushback all solverSetups under the same key
for ssName in solverSetups:
paramFile.dataDict.pushBack("solverSetup", ssName)
#Runner type
paramFile.dataDict.pushBack("runnerType", runnerType)
#Analysis dictionary
paramFile.dataDict.pushBack("analysis", DataDict())
paramFile.write()
#Create the staging folder
os.mkdir(os.path.join(folder, "stage"))
#Create the finished folder
os.mkdir(os.path.join(folder, "finished"))
#Create the analysis folder
os.mkdir(os.path.join(folder,"analysis"))
class AcdOptiMetaAnalysis:
"""
A meta-analysis is an analysis of several other anlysis;
i.e. "plotting" the exported output from analysis
or geometry/mesh variables against each other.
"""
instName = None
folder = None
collection = None
__paramfile = None
#Encodings describing where to get the x- or y variables.
# Meta-language syntax: {GEOM|MESH|ANA}.key.key([idx]) .... For ANA, field[1] is analysis name, field[2] is a name in exportResults
xVariable = None
yVariable = None
fVariable = None
fEquals = None
fLT = None
fGT = None
xArray = None
yArray = None
targetValue = None
lockdown = None
def __init__(self,folder,collection):
self.folder = folder
self.collection = collection
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
self.instName = instName = os.path.split(instName)[1]
#Load paramFile
self.__paramfile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'rw')
if self.__paramfile.dataDict["fileID"] != "AcdOptiMetaAnalysis":
raise AcdOptiException_metaAnalysis_loadFail("Got wrong fileID='" + self.__paramfile.dataDict["fileID"] + "'")
if self.__paramfile.dataDict.getValSingle("instName") != self.instName:
raise AcdOptiException_metaAnalysis_loadFail("instName doesn't match folder name")
self.lockdown = DataDict.boolconv(self.__paramfile.dataDict.getValSingle("lockdown"))
self.xVariable = self.__paramfile.dataDict["xVariable"]
self.yVariable = self.__paramfile.dataDict["yVariable"]
if len(self.__paramfile.dataDict.getVals("fVariable")) == 0:
self.__paramfile.dataDict.pushBack("fVariable", "")
self.__paramfile.dataDict.pushBack("fEquals", "")
self.__paramfile.dataDict.pushBack("fLT", "")
self.__paramfile.dataDict.pushBack("fGT", "")
self.__paramfile.write()
self.fVariable = self.__paramfile.dataDict["fVariable"]
def floatOrNone(strIn):
if strIn == "":
return None
else:
return float(strIn)
self.fEquals = floatOrNone(self.__paramfile.dataDict["fEquals"])
self.fGT = floatOrNone(self.__paramfile.dataDict["fGT"])
self.fLT = floatOrNone(self.__paramfile.dataDict["fLT"])
if len(self.__paramfile.dataDict.getVals("targetValue")) > 0:
self.targetValue = float(self.__paramfile.dataDict["targetValue"])
else:
self.targetValue = None
anaData = self.__paramfile.dataDict["anaData"]
self.xArray = []
self.yArray = []
for (x,y) in anaData:
self.xArray.append(float(x))
self.yArray.append(float(y))
#Final init & lockdown checks
if self.lockdown == True:
if self.yVariable == None or self.xVariable == None:
#The x,yArray may still be empty, don't check for this
raise AcdOptiException_metaAnalysis_loadFail("Lockdown, but xVariable='" + self.xVariable + "', yVariable='" + self.yVariable)
self.xArray = np.asarray(self.xArray)
self.yArray = np.asarray(self.yArray)
def write(self):
print "AcdOptiMetaAnalysis::write()"
self.__paramfile.dataDict.setValSingle("lockdown", str(self.lockdown))
self.__paramfile.dataDict.setValSingle("xVariable", self.xVariable)
self.__paramfile.dataDict.setValSingle("yVariable", self.yVariable)
self.__paramfile.dataDict.setValSingle("fVariable", self.fVariable)
def strFromFloatOrNone(fonIn):
if fonIn == None:
return ""
else:
return str(fonIn)
self.__paramfile.dataDict.setValSingle("fEquals",strFromFloatOrNone(self.fEquals))
self.__paramfile.dataDict.setValSingle("fGT",strFromFloatOrNone(self.fGT))
self.__paramfile.dataDict.setValSingle("fLT",strFromFloatOrNone(self.fLT))
if self.targetValue != None:
if len(self.__paramfile.dataDict.getVals("targetValue")) > 0:
self.__paramfile.dataDict.setValSingle("targetValue", str(self.targetValue))
else:
self.__paramfile.dataDict.pushBack("targetValue", str(self.targetValue))
else:
if len(self.__paramfile.dataDict.getVals("targetValue")) > 0:
self.__paramfile.dataDict.delItem("targetValue")
anaData = self.__paramfile.dataDict["anaData"]
anaData.clear()
for (x,y) in zip (self.xArray, self.yArray):
anaData.pushBack(str(x), str(y))
self.__paramfile.write()
def runAnalysis(self):
print "AcdOptiMetaAnalysis::runAnalysis()"
assert self.lockdown == False
#Do something...
xVar = self.xVariable.split(".")
yVar = self.yVariable.split(".")
if self.fEquals != None:
fVar = self.fVariable.split(".")
print "AcdOptiMetaAnalysis::runAnalysis() : xVar='" + str(xVar) + "', yVar='" + str(yVar) + "'"
# if not xVar[0] in ["GEOM","MESH","ANA"]:
# raise AcdOptiException_metaAnalysis_anaFail("xVar[0] invalid")
# if not yVar[0] in ["GEOM","MESH","ANA"]:
# raise AcdOptiException_metaAnalysis_anaFail("yVar[0] invalid")
#
for geom in self.collection.project.geomCollection.geomInstances.values():
xData = self.__getData(geom, xVar)
yData = self.__getData(geom, yVar)
if len(xData) > 1 and len(yData) > 1:
raise AcdOptiException_metaAnalysis_anaFail("Multiple hits for both xData and yData")
if self.fEquals != None:
fData = self.__getData(geom, fVar)
if len(xData) > 1 or len(yData) > 1 or len(fData) > 1:
raise AcdOptiException_metaAnalysis_anaFail("Multiple hits when filtering")
if fData[0] != self.fEquals:
continue
print xData, yData
if len(xData) > 1:
for x in xData:
self.xArray.append(x)
self.yArray.append(yData[0])
elif len(yData) > 1:
for y in yData:
self.xArray.append(xData[0])
self.yArray.append(y)
elif len(xData) == 1 and len(yData) == 1:
self.xArray.append(xData[0])
self.yArray.append(yData[0])
elif len(xData) == 0 or len(yData) == 0:
continue
else:
raise AcdOptiException_metaAnalysis_anaFail("This shouldn't happen?!")
#Icing on the top..
assert len(self.xArray) == len(self.yArray)
self.xArray = np.asarray(self.xArray)
self.yArray = np.asarray(self.yArray)
print self.xArray, self.yArray
self.lockdown = True
#self.write()
def __getData(self,geom,descriptor):
ret = []
if descriptor[0] == "GEOM":
if len(descriptor) > 2:
raise AcdOptiException_metaAnalysis_anaFail("Descriptor '" + str(descriptor) + "' starting with 'GEOM' too long")
if descriptor[1] in geom.templateOverrides_getKeys():
ret.append(float(geom.templateOverrides_get(descriptor[1])))
elif descriptor[1] in geom.template.paramDefaults_getKeys():
ret.append(float(geom.template.paramDefaults_get(descriptor[1])))
else:
raise AcdOptiException_metaAnalysis_anaFail("Geometry field ID '" + str(descriptor[1]) + "' not found")
elif descriptor[0] == "MESH":
if len(descriptor) > 2:
raise AcdOptiException_metaAnalysis_anaFail("Descriptor '" + str(descriptor) + "' starting with 'MESH' too long")
for mesh in geom.meshInsts.values():
if descriptor[1] in mesh.templateOverrides_getKeys():
ret.append(float(mesh.templateOverrides_get(descriptor[1])))
elif descriptor[1] in mesh.meshTemplate.paramDefaults_getKeys():
ret.append(float(mesh.meshTemplate.paramDefaults_get(descriptor[1])))
else:
raise AcdOptiException_metaAnalysis_anaFail("Mesh field ID '" + str(descriptor[1]) + "' not found")
elif descriptor[0] == "ANA":
for mesh in geom.meshInsts.values():
for rc in mesh.runConfigs.values():
#Get the ana
try:
ana = rc.analysis[descriptor[1]]
except KeyError:
continue
#Dig down into the exportResults
results = ana.exportResults
if len(descriptor) < 3:
raise AcdOptiException_metaAnalysis_anaFail("ANA descriptor '" + str(descriptor) + "' too short, doesn't specify field")
ret += self.__anaExportDigger(results, descriptor[2:])
else:
raise AcdOptiException_metaAnalysis_anaFail("Unknown descriptor[0]: '" + str(descriptor[0]) + "'")
return ret
def __anaExportDigger(self, exported, descriptor):
print "AcdOptiMetaAnalysis::__anaExportDigger() : descriptor='" + str(descriptor) + "', exported='" + str(exported) + "'"
ret = []
numberMatch = re.match("\w*\[([0-9]+)\]\Z",descriptor[0])
data = None
if numberMatch:
#next descriptor has an index
idx = int(numberMatch.group(1))
fetch = exported.getVals(descriptor[0].split("[")[0])
try:
data = [fetch[idx], ] #Put in a list to make loop work
except IndexError:
print "AcdOptiMetaAnalysis::__anaExportDigger() : WARNING - index not found!"
data = [] #If the
else:
data = exported.getVals(descriptor[0])
for val in data:
if type(val) == str:
if len(descriptor) > 1:
raise AcdOptiException_metaAnalysis_anaFail("Got deeper descriptors than data")
ret.append(val)
else: #DataDict
if len(descriptor) == 1:
raise AcdOptiException_metaAnalysis_anaFail("Got deeper data than descriptors")
ret += self.__anaExportDigger(val, descriptor[1:])
# if isinstance(val, DataDict):
# if not isMore:
# raise AcdOptiException_metaAnalysis_anaFail("Got deeper data than descriptors")
# ret += self.__anaExportDigger(val, descriptor[1:])
# else:
# if isMore:
# raise AcdOptiException_metaAnalysis_anaFail("Got deeper descriptors than data")
# ret.append(val)
return ret
def clearLockdown(self):
print "AcdOptiMetaAnalysis::clearLockdown()"
assert self.lockdown == True
self.xArray = []
self.yArray = []
self.lockdown = False
def exportData(self, fileName):
print "AcdOptiMetaAnalysis::exportData()"
assert self.lockdown == True
@staticmethod
def createNew(folder):
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
if os.path.isdir(folder):
raise AcdOptiException_metaAnalysis_createFail("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create paramfile
paramFile = AcdOptiFileParser_simple(os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "AcdOptiMetaAnalysis")
paramFile.dataDict.pushBack("instName", instName)
paramFile.dataDict.pushBack("xVariable", "")
paramFile.dataDict.pushBack("yVariable", "")
paramFile.dataDict.pushBack("fVariable", "")
paramFile.dataDict.pushBack("fEquals", "")
paramFile.dataDict.pushBack("fLT", "")
paramFile.dataDict.pushBack("fGT", "")
paramFile.dataDict.pushBack("anaData", DataDict()) #(key, value) = (x,y)
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.write()
def __init__(self,folder, project):
"""
Initialize (load) a GeometryCollection
stored in some folder
"""
self.folder = folder
self.project = project
#Load the param file
try:
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(self.folder, "paramFile.set"), 'rw')
except IOError:
raise AcdOptiException_geomCollection_loadFail\
("File paramFile.set not found")
if self.__paramFile.dataDict.getValSingle("fileID")\
!= "geomCollectionParamFile":
raise AcdOptiException_geomCollection_loadFail\
("Wrong fileID, got\""\
+ self.__paramFile.dataDict.getValSingle("fileID")\
+ "\" while loading __paramFile")
lock = self.__paramFile.dataDict.getValSingle("lockdown")
if lock == "True":
self.lockdown = True
elif lock == "False":
self.lockdown = False
else:
raise AcdOptiException_geomInstance_loadFail(\
"Invalid content in field \"lockdown\" of file paramFile.set")
#Load the default parameters
self.__paramDefaults = {}
try:
paramDefaults_data = self.__paramFile.dataDict.getValSingle("paramDefaults")
except AcdOptiException_dataDict_getValsSingle:
raise AcdOptiException_geomCollection_loadFail\
("Couldn't load paramDefaults from file paramFile.set")
if not isinstance(paramDefaults_data,DataDict):
raise AcdOptiException_geomCollection_loadFail\
("paramDefaults from paramFile is not a DataDict!")
for (k,v) in zip(paramDefaults_data.keys, paramDefaults_data.vals):
if k in self.__paramDefaults:
raise AcdOptiException_geomCollection_loadFail\
("Double occurence of key \"" + k + "\" in paramFile")
self.__paramDefaults[k] = v
#Load the template file
self.geomTemplateFile = AcdOptiCubitTemplateFile(os.path.join(folder,self.geomTemplateFile_name))
#Find subfolders and check if they are geometry instances
self.geomInstances = {}
for d in os.listdir(self.folder):
dAbs = os.path.abspath(os.path.join(self.folder,d))
if not os.path.isdir(dAbs):
#Skip files etc.
continue
try:
self.geomInstances[d] = AcdOptiGeometryInstance(dAbs, self)
except AcdOptiException_geomInstance_loadFail as e:
raise AcdOptiException_geomCollection_loadFail("Problem loading geometry instance \"" + d + "\", got error='" + str(e.args) + "'")
def __init__(self, folder, meshInstance):
print "AcdOptiRunConfig::__init__()"
self.folder = folder
self.meshInstance = meshInstance
self.status = "not_initialized"
#Construct the instance name from folder
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
self.instName = instname = os.path.split(instname)[1]
#Load the param file
try:
print os.path.join(self.folder, "paramFile.set")
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(self.folder, "paramFile.set"), 'rw')
except IOError:
raise AcdOptiException_runConfig_loadFail("File paramFile.set not found")
if self.__paramFile.dataDict.getValSingle("fileID") != "runConfigParamFile":
raise AcdOptiException_runConfig_loadFail\
("Wrong fileID, got \""+ self.__paramFile.dataDict.getValSingle("fileID")+"\" while loading paramFile")
if self.instName != self.__paramFile.dataDict.getValSingle("instName"):
raise AcdOptiException_runConfig_loadFail("instName doesn't match folder name, expected='"+self.instName+"', got '"+self.__paramFile.dataDict.getValSingle("instName")+"'")
self.status = self.__paramFile.dataDict.getValSingle("status")
if not self.status in self.statuses:
raise AcdOptiException_runConfig_loadFail("Status '" + self.status + "' not valid")
self.stageName = self.__paramFile.dataDict.getValSingle("stageName")
if self.stageName == "":
self.stageName = None
if self.stageName and self.status == "initialized":
raise AcdOptiException_runConfig_loadFail("StageName != None while status='" + self.status + "'")
#Generate stageFolder, stageFile, finishedFolder from stageName and status
if self.statuses.index(self.status) >= self.statuses.index("staged"):
self.stageFolder = os.path.join(self.folder,"stage",self.stageName)
if not os.path.isdir(self.stageFolder):
raise AcdOptiException_runConfig_loadFail("StageFolder '" + self.stageFolder + "' does not exist, but status='" + self.status + "'")
self.stageFile = os.path.join(self.folder, "stage", self.stageName) + ".tar.gz"
if not os.path.isfile(self.stageFile):
raise AcdOptiException_runConfig_loadFail("StageFile '" + self.stageFile + "' does not exist, but status='" + self.status + "'")
if self.status == "finished":
self.finishedFolder = os.path.join(self.folder, "finished", self.stageName)
if not os.path.isdir(self.finishedFolder):
raise AcdOptiException_runConfig_loadFail("FinishedFolder '" + self.finishedFolder + "' does not exist, but status='" + self.status + "'")
# self.stageFolder = self.__paramFile.dataDict.getValSingle("stageFolder")
# if self.stageFolder == "":
# self.stageFolder = None
#
# self.stageFile = self.__paramFile.dataDict.getValSingle("stageFile")
# if self.stageFile == "":
# self.stageFile = None
#
# self.finishedFolder = self.__paramFile.dataDict.getValSingle("finishedFolder")
# if self.finishedFolder == "":
# self.finishedFolder = None
# assert self.status != "finished"
#Sanity check on folders for staged- and finished data:
if not os.path.isdir(os.path.join(self.folder, "stage")):
raise AcdOptiException_runConfig_loadFail("stage folder not found")
if not os.path.isdir(os.path.join(self.folder, "finished")):
raise AcdOptiException_runConfig_loadFail("finished folder not found")
if not os.path.isdir(os.path.join(self.folder, "analysis")):
raise AcdOptiException_runConfig_loadFail("analysis folder not found")
#Load the solverSetups
solverSetupNames = self.__paramFile.dataDict.getVals("solverSetup")
self.solverSetups = []
for ssName in solverSetupNames:
self.solverSetups.append(AcdOptiSolverSetup(ssName, self))
#Load the runner
runnerType = self.__paramFile.dataDict["runnerType"]
self.runner = AcdOptiRunner.getRunner(runnerType, self)
#Load any analysis specified by the paramFile
anaDict = self.__paramFile.dataDict["analysis"]
self.analysis = {}
for (anaName, anaOptions) in anaDict:
print anaName, anaOptions
if anaName in self.analysis:
raise KeyError("Analysis name '" + anaName + "' encountered twice")
self.analysis[anaName] = AnalysisInterface.loadAnalysisByDict(anaOptions, os.path.join(self.folder, "analysis"), self)
#Refresh lockdowns
for solver in self.solverSetups:
solver.refreshLockdown()
self.runner.refreshLockdown()
self.runner.write()
#Check if we have an old file format, and if so write() to convert to new format
if len(self.__paramFile.dataDict.getVals("stageFolder")):
print "Updating to new file format without full paths specified (relocatable projects!)"
self.__paramFile.dataDict.delItem("stageFolder")
self.__paramFile.dataDict.delItem("stageFile")
self.__paramFile.dataDict.delItem("finishedFolder")
self.__paramFile.write()
class AcdOptiGeometryInstance():
"""
Class that organizes a specific geometry instance,
i.e. a set of geometry parameters,
which are merged with the template geometry to
create a CUBIT script. This script may be exported,
or sent to CUBIT for generating the solid.
Pointers to mesh instances are kept.
"""
def __init__(self, folder, template):
"""
Loads the geomInstance from the data
from folder.
Raises AcdOptiException_geomInstance_loadFail
if something (such as "not a geomInstance folder")
goes wrong.
"""
self.folder = folder
self.template = template
#Check that folder exists
if not os.path.isdir(folder):
raise AcdOptiException_geomInstance_loadFail(\
"Folder \"" + folder + "\" does not exist")
#Construct the instance name from folder name
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
self.instName = instName
#Load paramFile.set
try:
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'rw')
except IOError as e:
raise AcdOptiException_geomInstance_loadFail(\
"Problem loading file \"paramFile.set\" in folder \"" +\
folder + "\", got an IOError")
if self.__paramFile.dataDict.getValSingle("fileID") != "geomInstance":
raise AcdOptiException_geomInstance_loadFail(
"Wrong fileID in geomInstance.set, got \"" +
self.__paramFile.dataDict.getValSingle("fileID") + "\"")
lock = self.__paramFile.dataDict.getValSingle("lockdown")
if lock == "True":
self.lockdown = True
elif lock == "False":
self.lockdown = False
else:
raise AcdOptiException_geomInstance_loadFail(\
"Invalid content in field \"lockdown\" of file paramFile.set")
#Find and load template overrides
self.__templateOverrides = {}
try:
templateOverrides_data = self.__paramFile.dataDict.getValSingle("templateOverrides")
except AcdOptiException_dataDict_getValsSingle:
raise AcdOptiException_geomInstance_loadFail\
("Couldn't load templateOverrides from paramFile.set")
if not isinstance(templateOverrides_data,DataDict):
raise AcdOptiException_geomInstance_loadFail\
("templateOverrides from paramFile.set is not a DataDict!")
for (k,v) in zip(templateOverrides_data.keys, templateOverrides_data.vals):
if k in self.__templateOverrides:
raise AcdOptiException_geomInstance_loadFail\
("Double occurrence of key \"" + k + "\" in templateOverrides on file")
if not k in self.template.paramDefaults_getKeys():
raise AcdOptiException_geomInstance_loadFail(
"Entry \"" + k + "\" in templateOverrides on file has no match in the template")
self.__templateOverrides[k] = v
#Are there any mesh instances?
self.meshInsts = {}
for d in os.listdir(os.path.join(self.folder,self.meshInstanceFolderName)):
dAbs = os.path.abspath(os.path.join(folder,self.meshInstanceFolderName,d))
if not os.path.isdir(dAbs):
#Skip files etc.
continue
#try:
self.meshInsts[d] = AcdOptiMeshInstance(dAbs, self, self.template.project.meshTemplateCollection)
#except AcdOptiException_meshInstance_loadFail as e:
# raise AcdOptiException_geomCollection_loadFail(\
# "Problem loading mesh instance \"" + dAbs + "\", got error \"" + str(e) + "\"")
#Scan instance name
#self.scanInstance_name = self.__paramFile.dataDict["scanInstance_name"]
self.scanInstances = []
def write(self):
"""
Update the contents of self.__paramFile to reflect
current status of the class
"""
print "AcdOptiGeometryInstance::write()"
self.__paramFile.dataDict.setValSingle("lockdown", str(self.lockdown))
# if self.scanInstance != None:
# self.__paramFile.dataDict.setValSingle("scanInstance_name", self.scanInstance.instName)
# else:
# self.__paramFile.dataDict.setValSingle("scanInstance_name", "")
templateOverrides_data = self.__paramFile.dataDict.getValSingle("templateOverrides")
templateOverrides_data.clear()
for k in self.__templateOverrides:
templateOverrides_data.pushBack(k,self.__templateOverrides[k])
self.__paramFile.write()
def generatePattern(self):
"""
Generate substitution dict
"""
params = self.template.paramDefaults_copy()
for k in self.__templateOverrides:
assert k in params
params[k] = self.__templateOverrides[k]
return params
def generateCubitJou(self):
"""
Uses the currently defined variables (overrides + inherited)
to generate a cubit script. Returns a tuple with
the generated script as a string, and a list of tag's not found in the template.
"""
params = self.generatePattern()
#Get the CUBIT template file object
#templateFile = self.template.geomTemplateFile
#return templateFile.subsString(params)
(runScript, notFound) = self.template.geomTemplateFile.generateJou(params, self.cubitGeomPreCommands, self.cubitGeomPostCommands)
ret = ""
for line in runScript:
ret += line + "\n"
return (ret, notFound)
def generateGeometry(self):
"""
Generates the CUBIT journal
and runs this with the pre- and post-commands that are useful
for generating a solid.
Also saves the journal as geom.jou for later reference.
Side-effect: Sets lockdown
Returns any keys not found.
"""
#Preparation
params = self.generatePattern()
templateFile = self.template.geomTemplateFile
#Make the journal for later reference
refJouFileName = os.path.join(self.folder, "geom.jou")
if os.path.isfile(refJouFileName):
os.remove(refJouFileName)
templateFile.writeJouFile(params,refJouFileName,self.cubitGeomPreCommands, self.cubitGeomPostCommands)
#Generate the geometry
templateFile.runCubit(params, preCommands=self.cubitGeomPreCommands, postCommands=self.cubitGeomPostCommands)
os.rename("geom.cub", os.path.join(self.folder, "geom.cub"))
self.setLockdown()
def addMeshInstance(self, meshTemplateName, meshInstanceName):
"""
Creates and adds a new meshInstance to this geometryInstance.
Raises an AcdOptiException_geomInstance_nameError
if the meshInstanceName is already in use or the meshTemplate does not exist.
"""
print "AcdOptiGeometryInstance::addMeshInstance(meshTemplateName=\""\
+ meshTemplateName + "\" , meshInstanceName=\"" + meshInstanceName + "\" )"
#Check that the mesh instance name is not already in use:
if meshInstanceName in self.meshInsts:
raise AcdOptiException_geomInstance_nameError("Mesh instance name already in use")
folder = os.path.join(self.folder,self.meshInstanceFolderName,meshInstanceName)
try:
AcdOptiMeshInstance.createNew(folder, self.instName, meshTemplateName)
except AcdOptiException_meshInstance_createFail:
raise AcdOptiException_geomInstance_nameError("Mesh instance name \"" + meshInstanceName + "\" already in use")
meshInstance = AcdOptiMeshInstance(folder, self, self.template.project.meshTemplateCollection)
self.meshInsts[meshInstanceName] = meshInstance
print "AcdOptiGeometryInstance::addMeshInstance():Mesh instances currently loaded:" + str(self.meshInsts)
def setLockdown(self):
"""
Set lockdown = True,
indicating that a geometry has been generated, and that
the templateOverrides should not be touched.
Also writes instance to file.
"""
print "AcdOptiGeometryInstance::setLockdown()"
self.template.setLockdown()
self.lockdown = True
self.write()
def clearLockdown(self):
"""
Clears lockdown of the geometry instance and any mesh instances,
and deletes generated cubit objects.
Also writes instance to file.
"""
print "AcdOptiGeometryInstance::clearLockdown()"
if not self.lockdown:
return
for mesh in self.meshInsts.values():
mesh.clearLockdown()
if os.path.isfile(os.path.join(self.folder, "geom.cub")):
os.remove(os.path.join(self.folder, "geom.cub"))
if os.path.isfile(os.path.join(self.folder, "geom.jou")):
os.remove(os.path.join(self.folder, "geom.jou"))
self.lockdown = False
self.write()
def templateOverrides_insert(self,key,val):
if self.lockdown:
raise AcdOptiException_geomInstance_lockdownError
self.__templateOverrides[key] = val
return val
def templateOverrides_get(self,key):
return self.__templateOverrides[key]
def templateOverrides_getKeys(self):
return self.__templateOverrides.keys()
def templateOverrides_len(self):
return len(self.__templateOverrides)
def templateOverrides_del(self,key):
if self.lockdown:
raise AcdOptiException_geomInstance_lockdownError
val = self.__templateOverrides[key]
del self.__templateOverrides[key]
return val
def templateOverrides_clear(self):
if self.lockdown:
raise AcdOptiException_geomInstance_lockdownError
self.__templateOverrides.clear()
@staticmethod
def createNew(folder):
"""
Creates a new empty geomInstance in a
not previously existing folder.
Geometry instance name will be the same as the folder name.
Raises AcdOptiException_geomInstance_createFail
is something goes wrong (such as "Folder already exists")
"""
#Construct the instance name from folder
instName = folder
if instName[-1] == "/":
instName = instName[0:-1]
instName = os.path.split(instName)[1]
#Create the folder
if os.path.isdir(folder):
raise AcdOptiException_geomInstance_createFail("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create paramFile.set file
paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "geomInstance")
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.dataDict.pushBack("templateOverrides", DataDict())
#paramFile.dataDict.pushBack("scanInstance_name", "")
paramFile.write()
#Create folder for meshInstance's
os.mkdir(os.path.join(folder, AcdOptiGeometryInstance.meshInstanceFolderName))
@staticmethod
def createNew_clone(folder, cloneFrom):
"""
Creates a new geomInstance in a not previously existing folder,
which has identical settings as an already existing geometryInstance.
The newly created geomInstance is then returned.
This is a deep copy, meshInstances etc. are also cloned.
"""
#Create the new geomInstance
AcdOptiGeometryInstance.createNew(folder)
newInstance = AcdOptiGeometryInstance(folder, cloneFrom.template)
#Copy information
for key in cloneFrom.templateOverrides_getKeys():
newInstance.templateOverrides_insert(key, cloneFrom.templateOverrides_get(key))
for (meshName, mesh) in cloneFrom.meshInsts.iteritems():
newMesh = AcdOptiMeshInstance.createNew_clone(os.path.join(folder,"meshInstances",meshName), mesh, newInstance)
newInstance.meshInsts[meshName] = newMesh
newInstance.write()
return newInstance
def cloneMeshInstance(self, oldObject, newName):
"""
Clone a mesh object into a new name and attach it to the current geometryInstance
"""
print "AcdOptiGeometryInstance::cloneMeshInstance()"
assert not newName in self.meshInsts
newMesh = AcdOptiMeshInstance.createNew_clone(os.path.join(self.folder, self.meshInstanceFolderName, newName), oldObject, self)
self.meshInsts[newName] = newMesh
self.write()
#Object variables
lockdown = False #Write-protected?
folder = None #Folder where this geometryInstance is stored
__paramFile = None #Settings file for the geometryInstance
instName = None #Name of this instance
meshInsts = None #Dict of mesh instances for this geometry
template = None #Pointer to the GeometryCollection of which
# this is an instance
__templateOverrides = None #Dict describing the variables that are
# given a different value than the default
# from the collection.
scanInstances = None #A list of the scans this geomInstances belongs to.
# This is set by the scanInstance's __init__()
# The list may contain both old- and new-style scans
#Static class fields
cubitGeomPreCommands = ["reset"]
cubitGeomPostCommands = ["save as 'geom.cub' overwrite"]
meshInstanceFolderName = "meshInstances"
class AcdOptiGeometryCollection:
"""
Class that knows what the geometry parameters are,
and organizes the content of the geomInstances folder.
"""
#Object variables
geomInstances = None
folder = None
__paramDefaults = None #Default value of the parameters
__paramFile = None #File holding these
geomTemplateFile = None
geomTemplateFile_name = "geomTemplateFile.jou.template"
lockdown = False
project = None #Pointer to the top-level AcdOptiProject
def __init__(self,folder, project):
"""
Initialize (load) a GeometryCollection
stored in some folder
"""
self.folder = folder
self.project = project
#Load the param file
try:
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(self.folder, "paramFile.set"), 'rw')
except IOError:
raise AcdOptiException_geomCollection_loadFail\
("File paramFile.set not found")
if self.__paramFile.dataDict.getValSingle("fileID")\
!= "geomCollectionParamFile":
raise AcdOptiException_geomCollection_loadFail\
("Wrong fileID, got\""\
+ self.__paramFile.dataDict.getValSingle("fileID")\
+ "\" while loading __paramFile")
lock = self.__paramFile.dataDict.getValSingle("lockdown")
if lock == "True":
self.lockdown = True
elif lock == "False":
self.lockdown = False
else:
raise AcdOptiException_geomInstance_loadFail(\
"Invalid content in field \"lockdown\" of file paramFile.set")
#Load the default parameters
self.__paramDefaults = {}
try:
paramDefaults_data = self.__paramFile.dataDict.getValSingle("paramDefaults")
except AcdOptiException_dataDict_getValsSingle:
raise AcdOptiException_geomCollection_loadFail\
("Couldn't load paramDefaults from file paramFile.set")
if not isinstance(paramDefaults_data,DataDict):
raise AcdOptiException_geomCollection_loadFail\
("paramDefaults from paramFile is not a DataDict!")
for (k,v) in zip(paramDefaults_data.keys, paramDefaults_data.vals):
if k in self.__paramDefaults:
raise AcdOptiException_geomCollection_loadFail\
("Double occurence of key \"" + k + "\" in paramFile")
self.__paramDefaults[k] = v
#Load the template file
self.geomTemplateFile = AcdOptiCubitTemplateFile(os.path.join(folder,self.geomTemplateFile_name))
#Find subfolders and check if they are geometry instances
self.geomInstances = {}
for d in os.listdir(self.folder):
dAbs = os.path.abspath(os.path.join(self.folder,d))
if not os.path.isdir(dAbs):
#Skip files etc.
continue
try:
self.geomInstances[d] = AcdOptiGeometryInstance(dAbs, self)
except AcdOptiException_geomInstance_loadFail as e:
raise AcdOptiException_geomCollection_loadFail("Problem loading geometry instance \"" + d + "\", got error='" + str(e.args) + "'")
def write(self):
"""
Write the current contents of this class to paramFile,
which is then written to file.
"""
print "AcdOptiGeometryCollection::write()"
print "Lockdown =", self.lockdown
self.__paramFile.dataDict.setValSingle("lockdown", str(self.lockdown))
paramDefaults_data = self.__paramFile.dataDict.getValSingle("paramDefaults")
paramDefaults_data.clear()
for (k,v) in self.__paramDefaults.iteritems():
paramDefaults_data.pushBack(k,v)
print self.__paramFile
self.__paramFile.write()
def mayDelete(self,key):
"""
Check if key can be deleted -
if it has it been overridden in a GeometryInstance
or (TODO) is used in the cubit template script,
return the problematic AcdOptiGeometryInstance or AcdOptiCubitTemplateFile.
Normally return None, meaning that the key may be deleted.
"""
for gi in self.geomInstances:
if key in self.geomInstances[gi].templateOverrides_getKeys():
return self.geomInstances[gi]
return None
def addGeomInstance(self,name):
"""
Creates and adds a GeometryInstance to the geomCollection.
Raises an AcdOptiException_geomInstance_createFail if the name already exists.
The created geomInstance is returned.
"""
#Create the AcdOptiGeometryInstance
folder = os.path.join(self.folder, name)
AcdOptiGeometryInstance.createNew(folder)
#Add it to the project/geomCollection
assert not name in self.geomInstances, "Folder did not exist but there was an entry in self.geomInstances?!? name='" + name + "'"
self.geomInstances[name] = AcdOptiGeometryInstance(folder,self)
return self.geomInstances[name]
def cloneGeomInstance(self,oldName, newName):
"""
Create a new geomInstance called <newName>, having
identical same settings, meshInstance etc. as the one called <oldName>.
The new geom instance is returned.
"""
assert oldName in self.geomInstances
newGeom = AcdOptiGeometryInstance.createNew_clone(os.path.join(self.folder, newName), self.geomInstances[oldName])
self.geomInstances[newName] = newGeom
return newGeom
def setLockdown(self):
"""
Set lockdown = True,
indicating that a geometry instance has been generated, and that
the template should not be touched.
Also writes data to file
"""
print "AcdOptiGeometryCollection::setLockdown()"
self.lockdown = True
self.write()
def clearLockdown(self):
"""
Clears lockdown of the geometry collection and
any geometry/mesh instances, deleting generated cubit objects
Also writes instance to file.
"""
print "AcdOptiGeometryCollection::clearLockdown()"
for geomInstance in self.geomInstances.values():
geomInstance.clearLockdown()
self.lockdown = False
self.write()
def paramDefaults_insert(self,key,val):
if self.lockdown:
raise AcdOptiException_geomCollection_lockdownError
self.__paramDefaults[key] = val
return val
def paramDefaults_get(self,key):
return self.__paramDefaults[key]
def paramDefaults_getKeys(self):
return self.__paramDefaults.keys()
def paramDefaults_copy(self):
return self.__paramDefaults.copy()
def paramDefaults_len(self):
return len(self.__paramDefaults)
def paramDefaults_del(self,key):
if self.lockdown:
raise AcdOptiException_geomCollection_lockdownError
val = self.__paramDefaults[key]
del self.__paramDefaults[key]
return val
def paramDefaults_clear(self):
if self.lockdown:
raise AcdOptiException_geomCollection_lockdownError
self.__paramDefaults.clear()
@staticmethod
def createNew(folder):
"""
Sets up the basic structure inside
a given folder, which is created.
"""
#Create the directory
os.mkdir(folder)
#File that holds the parameters and their default values
paramFile = AcdOptiFileParser_simple(\
os.path.join(folder,"paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "geomCollectionParamFile")
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.dataDict.pushBack("paramDefaults",DataDict())
paramFile.write()
#Default empty template file
AcdOptiCubitTemplateFile.createEmpty(os.path.join(folder,AcdOptiGeometryCollection.geomTemplateFile_name))
class AcdOptiMeshTemplate:
"""
Class that represents a specific
mesh template script, which can be applied
in different configurations to different
geometry instances.
One mesh template =
meshing script with variables + default values for those variables.
"""
def __init__(self, folder):
self.folder = folder
#Construct the instance name from folder
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
self.instName = instname = os.path.split(instname)[1]
#Load the param file
try:
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(self.folder, "paramFile.set"), 'rw')
except IOError:
raise AcdOptiException_meshTemplate_loadFail("File paramFile.set not found")
if self.__paramFile.dataDict.getValSingle("fileID")\
!= "meshTemplateParamFile":
raise AcdOptiException_meshTemplate_loadFail\
("Wrong fileID, got \""\
+ self.__paramFile.dataDict.getValSingle("fileID")\
+ "\" while loading paramFile")
if self.__paramFile.dataDict.getValSingle("instName")\
!= instname:
raise AcdOptiException_meshTemplate_loadFail("templateName doesn't match folder name")
lock = self.__paramFile.dataDict.getValSingle("lockdown")
if lock == "True":
self.lockdown = True
elif lock == "False":
self.lockdown = False
else:
raise AcdOptiException_meshTemplate_loadFail(\
"Invalid content in field \"lockdown\" of file paramFile.set, got'"+lock+"'")
#Load the default parameters
self.__paramDefaults = {}
try:
paramDefaults_data = self.__paramFile.dataDict.getValSingle("paramDefaults")
except AcdOptiException_dataDict_getValsSingle:
raise AcdOptiException_meshTemplate_loadFail\
("Couldn't load paramDefaults from file paramFile.set")
if not isinstance(paramDefaults_data,DataDict):
raise AcdOptiException_meshTemplate_loadFail\
("paramDefaults from paramFile is not a DataDict!")
for (k,v) in zip(paramDefaults_data.keys, paramDefaults_data.vals):
if k in self.__paramDefaults:
raise AcdOptiException_meshTemplate_loadFail\
("Double occurrence of key \"" + k + "\" in paramFile")
self.__paramDefaults[k] = v
#Load the template file
self.meshTemplateFile = AcdOptiCubitTemplateFile(os.path.join(folder,self.meshTemplateFile_name))
#Initialize __meshInstances
self.__meshInstances = []
def mayDelete(self,key): #TODO!!
"""
Check if key can be deleted -
if it has it been overridden in a MeshInstance
or (TODO) is used in the cubit template script,
return the problematic AcdOptiMeshInstance or AcdOptiCubitTemplateFile.
Normally return None, meaning that the key may be deleted.
"""
print "AcdOptiMeshTemplate::mayDelete()"
for mi in self.__meshInstances:
if key in mi.templateOverrides_getKeys():
return mi
return None
def write(self):
"""
Write the current contents of this class to paramFile,
which is written to file.
"""
print "AcdOptiMeshTemplate::write()"
self.__paramFile.dataDict.setValSingle("lockdown", str(self.lockdown))
paramDefaults_data = self.__paramFile.dataDict.getValSingle("paramDefaults")
paramDefaults_data.clear()
for (k,v) in self.__paramDefaults.iteritems():
paramDefaults_data.pushBack(k,v)
self.__paramFile.write()
def registerInstance(self, instance):
"""
Register a mesh instance with this mesh template.
"""
self.__meshInstances.append(instance)
def setLockdown(self):
"""
Set lockdown = True,
indicating that a mesh instance has been generated, and that
the template should not be touched.
Also writes data to file.
"""
print "AcdOptMeshTemplate::setLockdown()"
self.lockdown = True
self.write()
def clearLockdown(self):
"""
Clears lockdown of the mesh template and
any mesh instances, deleting generated cubit objects.
Also writes instance to file.
"""
print "AcdOptiMeshTemplate::clearLockdown()"
for meshInstance in self.__meshInstances:
meshInstance.clearLockdown()
self.lockdown = False
self.write()
def paramDefaults_insert(self,key,val):
if self.lockdown:
raise AcdOptiException_meshTemplate_lockdownError
self.__paramDefaults[key] = val
return val
def paramDefaults_get(self,key):
return self.__paramDefaults[key]
def paramDefaults_getKeys(self):
return self.__paramDefaults.keys()
def paramDefaults_copy(self):
return self.__paramDefaults.copy()
def paramDefaults_len(self):
return len(self.__paramDefaults)
def paramDefaults_del(self,key):
if self.lockdown:
raise AcdOptiException_meshTemplate_lockdownError
val = self.__paramDefaults[key]
del self.__paramDefaults[key]
return val
def paramDefaults_clear(self):
if self.lockdown:
raise AcdOptiException_meshTemplate_lockdownError
self.__paramDefaults.clear()
@staticmethod
def createNew(folder):
"""
Creates a new empty meshTemplate in a
not previously existing folder.
Folder name should be the same as geometry instance name.
Raises AcdOptiException_meshTemplate_createFail
is something goes wrong (such as "Folder already exists")
"""
#Construct the instance name from folder
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
instname = os.path.split(instname)[1]
if os.path.isdir(folder):
raise AcdOptiException_meshTemplate_createFail ("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create the paramFile
paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "meshTemplateParamFile")
paramFile.dataDict.pushBack("instName", instname)
paramFile.dataDict.pushBack("lockdown", "False")
paramFile.dataDict.pushBack("paramDefaults", DataDict())
paramFile.write()
#Default empty template file
AcdOptiCubitTemplateFile.createEmpty(os.path.join(folder,AcdOptiMeshTemplate.meshTemplateFile_name))
#Object variables
folder = None #Folder where this template lives
instName = None #Name of this template
meshTemplateFile_name = "meshTemplateFile.jou.template"
meshTemplateFile = None
__meshInstances = None #List of mesh instances implementing this template
lockdown = False
__paramFile = None
__paramDefaults = None
def createNew(folder):
os.mkdir(folder)
paramFile = AcdOptiFileParser_simple(os.path.join(folder,"paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "AcdOptiScanCollection")
paramFile.write()
class AcdOptiRunConfig:
"""
Class representing a run configuration,
holding solvers and job setup, setups and allows starting/stopping runs
and querying their status.
"""
folder = None
instName = None
#lockdown = None
solverSetups = None #List of solverSetups
runner = None #The runner in use
meshInstance = None #Pointer back to the "owning" meshInstance
__paramFile = None #Settings file
status = "not_initialized"
stageName = None #status > staged, name of folder or tar.gz file (without ending) with staged data (else None)
stageFolder = None #Path to the staging directory (if status>staged, else None)
stageFile = None #Full path to the tar.gz file with the staged data (if status>staged, else None)
finishedFolder = None #Full path to the folder with the finished data
analysis = None #Dict of the analysis loaded
statuses= ["not_initialized", # Before object is fully created
"initialized", # Has one or more runners and a meshInstance
"staged", # Object is staged, all files are in place
"remote::uploaded",# Files uploaded to HPC, but has not yet appeared in queue system
"remote::queued", # Job is submitted to HPC, but has not started yet
"remote::running", # Job is running
"remote::unclean", # Job was submitted, but crashed or was canceled
"remote::finished",# Job has finished, but is not downloaded
"local::running", # Local job is running
"finished"] # Job is completely finished, ready for analysis
def __init__(self, folder, meshInstance):
print "AcdOptiRunConfig::__init__()"
self.folder = folder
self.meshInstance = meshInstance
self.status = "not_initialized"
#Construct the instance name from folder
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
self.instName = instname = os.path.split(instname)[1]
#Load the param file
try:
print os.path.join(self.folder, "paramFile.set")
self.__paramFile = AcdOptiFileParser_simple(\
os.path.join(self.folder, "paramFile.set"), 'rw')
except IOError:
raise AcdOptiException_runConfig_loadFail("File paramFile.set not found")
if self.__paramFile.dataDict.getValSingle("fileID") != "runConfigParamFile":
raise AcdOptiException_runConfig_loadFail\
("Wrong fileID, got \""+ self.__paramFile.dataDict.getValSingle("fileID")+"\" while loading paramFile")
if self.instName != self.__paramFile.dataDict.getValSingle("instName"):
raise AcdOptiException_runConfig_loadFail("instName doesn't match folder name, expected='"+self.instName+"', got '"+self.__paramFile.dataDict.getValSingle("instName")+"'")
self.status = self.__paramFile.dataDict.getValSingle("status")
if not self.status in self.statuses:
raise AcdOptiException_runConfig_loadFail("Status '" + self.status + "' not valid")
self.stageName = self.__paramFile.dataDict.getValSingle("stageName")
if self.stageName == "":
self.stageName = None
if self.stageName and self.status == "initialized":
raise AcdOptiException_runConfig_loadFail("StageName != None while status='" + self.status + "'")
#Generate stageFolder, stageFile, finishedFolder from stageName and status
if self.statuses.index(self.status) >= self.statuses.index("staged"):
self.stageFolder = os.path.join(self.folder,"stage",self.stageName)
if not os.path.isdir(self.stageFolder):
raise AcdOptiException_runConfig_loadFail("StageFolder '" + self.stageFolder + "' does not exist, but status='" + self.status + "'")
self.stageFile = os.path.join(self.folder, "stage", self.stageName) + ".tar.gz"
if not os.path.isfile(self.stageFile):
raise AcdOptiException_runConfig_loadFail("StageFile '" + self.stageFile + "' does not exist, but status='" + self.status + "'")
if self.status == "finished":
self.finishedFolder = os.path.join(self.folder, "finished", self.stageName)
if not os.path.isdir(self.finishedFolder):
raise AcdOptiException_runConfig_loadFail("FinishedFolder '" + self.finishedFolder + "' does not exist, but status='" + self.status + "'")
# self.stageFolder = self.__paramFile.dataDict.getValSingle("stageFolder")
# if self.stageFolder == "":
# self.stageFolder = None
#
# self.stageFile = self.__paramFile.dataDict.getValSingle("stageFile")
# if self.stageFile == "":
# self.stageFile = None
#
# self.finishedFolder = self.__paramFile.dataDict.getValSingle("finishedFolder")
# if self.finishedFolder == "":
# self.finishedFolder = None
# assert self.status != "finished"
#Sanity check on folders for staged- and finished data:
if not os.path.isdir(os.path.join(self.folder, "stage")):
raise AcdOptiException_runConfig_loadFail("stage folder not found")
if not os.path.isdir(os.path.join(self.folder, "finished")):
raise AcdOptiException_runConfig_loadFail("finished folder not found")
if not os.path.isdir(os.path.join(self.folder, "analysis")):
raise AcdOptiException_runConfig_loadFail("analysis folder not found")
#Load the solverSetups
solverSetupNames = self.__paramFile.dataDict.getVals("solverSetup")
self.solverSetups = []
for ssName in solverSetupNames:
self.solverSetups.append(AcdOptiSolverSetup(ssName, self))
#Load the runner
runnerType = self.__paramFile.dataDict["runnerType"]
self.runner = AcdOptiRunner.getRunner(runnerType, self)
#Load any analysis specified by the paramFile
anaDict = self.__paramFile.dataDict["analysis"]
self.analysis = {}
for (anaName, anaOptions) in anaDict:
print anaName, anaOptions
if anaName in self.analysis:
raise KeyError("Analysis name '" + anaName + "' encountered twice")
self.analysis[anaName] = AnalysisInterface.loadAnalysisByDict(anaOptions, os.path.join(self.folder, "analysis"), self)
#Refresh lockdowns
for solver in self.solverSetups:
solver.refreshLockdown()
self.runner.refreshLockdown()
self.runner.write()
#Check if we have an old file format, and if so write() to convert to new format
if len(self.__paramFile.dataDict.getVals("stageFolder")):
print "Updating to new file format without full paths specified (relocatable projects!)"
self.__paramFile.dataDict.delItem("stageFolder")
self.__paramFile.dataDict.delItem("stageFile")
self.__paramFile.dataDict.delItem("finishedFolder")
self.__paramFile.write()
def refreshStatus(self):
"""
Refresh async statuses "remote::*" and "local::*"
by calling the runner.
Also switch between not_initialized and initialized
by checking runner and solverSetups.
"""
print "AcdOptiRunConfig::refreshStatus()"
if not self.status in self.statuses:
raise AcdOptiException_runConfig_updateStateError("Not a valid status, current status='" + self.status + "'")
#Switch initialized/not_initialized
# TODO: more thorough checks...
if self.status == "not_initialized":
if len(self.solverSetups):
self.status = "initialized"
return
elif self.status == "initialized":
if not len(self.solverSetups):
self.status = "not_initialized"
return
#Check remote statuses
if not (self.status.startswith("remote::") or self.status.startswith("local::")):
raise AcdOptiException_runConfig_updateStateError("Not an async status, current status='" + self.status + "'")
status = self.runner.queryStatus()
if not status in self.statuses:
raise AcdOptiException_runConfig_updateStateError("Got an invalid status, current status='" + self.status + "'")
self.status = status
#Update solvers
for solver in self.solverSetups:
solver.refreshLockdown()
self.runner.refreshLockdown()
self.write()
def stage(self):
"""
Stages all the files needed for running the job
"""
if self.status == "staged":
return #Skip if already staged
elif self.status != "initialized":
raise AcdOptiException_runConfig_stageError("Not in status 'initialized', status='" + self.status + "'")
#Find the correct name to use for the staging and create the folder
meshInstance = self.meshInstance
geomInstance = meshInstance.geometryInstance
project = geomInstance.template.project
meshTemplate = meshInstance.meshTemplate
solverString = ""
for solver in self.solverSetups:
solverString += solver.name + "-"
solverString = solverString[:-5]
# self.stageName = "AcdOpti-stage--" + project.projectName_name + "-"\
# + geomInstance.instName + "-" + meshTemplate.instName + "--"\
# + meshInstance.instName + "--"\
# + self.instName + "-" + self.runner.type + "-" + solverString + "--"\
# + datetime.now().isoformat().replace(":","-") #Avoid ":" at all cost: ACD doesn't like them in their (auto-generated) input files
self.stageName = project.projectName_name + "-"\
+ geomInstance.instName + "-"\
+ meshInstance.instName + "--"\
+ datetime.now().isoformat().replace(":","-") #Avoid ":" at all cost: ACD doesn't like them in their (auto-generated) input files
self.stageFolder = os.path.join(self.folder,"stage",self.stageName)
os.mkdir(self.stageFolder)
#Get the mesh file, generate if necessary
if not self.meshInstance.lockdown:
try:
self.meshInstance.generateMesh()
except AcdOptiException_meshInstance_generateFail as e:
self.clearLockdown(forced=True)
raise AcdOptiException_runConfig_stageError("Error when generating mesh:\n" + e.args[0])
shutil.copy(os.path.join(self.meshInstance.folder, "mesh.ncdf"), self.stageFolder)
shutil.copy(os.path.join(self.meshInstance.folder, "mesh.jou" ), self.stageFolder)
#Get the geometry journal file (for reference)
shutil.copy(os.path.join(self.meshInstance.geometryInstance.folder, "geom.jou"), self.stageFolder)
#Prepare SolverSetups
for solv in self.solverSetups:
solv.stage()
#Prepare runner
try:
self.runner.stage()
except AcdOptiException_optiRunner_stageError as e:
#Something went wrong when staging the runner
# cleanup and raise exception
self.clearLockdown(forced=True)
raise AcdOptiException_runConfig_stageError("Error when staging runner:\n" + e.args[0])
#Zip the folder to make it ready for upload
self.stageFile = os.path.join(self.folder, "stage", self.stageName) + ".tar.gz"
stageFileObject = tarfile.open(self.stageFile, mode="w:gz")
stageFileObject.add(self.stageFolder, arcname=self.stageName)
stageFileObject.close()
#Set status flag
self.status = "staged"
for solver in self.solverSetups:
solver.refreshLockdown()
self.runner.refreshLockdown()
self.write()
def upload(self):
"""
Given that everything is staged, and we have a "remote" runner,
upload the data to the HPC
"""
assert self.status=="staged"
assert self.runner.isRemote()
self.runner.upload()
self.status = "remote::uploaded"
self.write()
def run(self):
"""
Given that the runConfig etc. is ready to run, starts the run.
"""
print "AcdOptiRunConfig::run()"
if self.runner.isRemote():
assert self.status == "remote::uploaded"
self.runner.run()
self.status = "remote::queued"
self.write()
def getRemote(self):
"""
Given that there are remote data and we are not running,
download this data to the local machine.
"""
print "AcdOptiRunConfig::getRemote()"
assert self.runner.isRemote()
assert self.status == "remote::finished" or self.status == "remote::unclean"
self.finishedFolder = self.runner.getRemoteData()
self.remoteCleanup()
self.status = "finished"
self.write()
def cancel(self):
"""
Given that there is a run in progress or queued, cancel it.
"""
print "AcdOptiRunConfig::cancel()"
if self.runner.isRemote():
assert self.status == "remote::running" or\
self.status == "remote::queued"
self.runner.cancelRun()
self.status = "remote::unclean"
self.write()
def remoteCleanup(self):
print "AcdOptiRunConfig::remoteCleanup()"
assert self.runner.isRemote()
assert self.status=="remote::uploaded" or self.status=="remote::finished" or self.status=="remote::unclean" or self.status=="finished"
self.runner.remoteCleanup()
if self.status != "finished":
self.status = "staged"
self.write()
def clearLockdown(self,forced=False):
"""
Clears the staged data and any solutions/analysis
results that might exist, bringing status back to 'initialized'"
Set forced=True if the clearLockdown() happens because of a crash in stage,
and is intended to bring the runConfig back to a defined state
"""
print "AcdOptiRunConfig::clearLockdown()"
if self.status == "initialized" and not forced:
return
#Clear lockdown of analysis
for ana in self.analysis.itervalues():
print ana.instName
ana.clearLockdown()
#Clear staged data folder and tarball
if self.stageFolder and os.path.isdir(self.stageFolder):
for d in os.listdir(self.stageFolder):
dAbs = os.path.abspath(os.path.join(self.stageFolder,d))
os.remove(dAbs)
os.rmdir(self.stageFolder)
if self.stageFile and os.path.isfile(self.stageFile):
os.remove(self.stageFile)
self.stageName = None
self.stageFolder = None
self.stageFile = None
#Clear finished data
shutil.rmtree(os.path.join(self.folder, "finished"))
os.mkdir(os.path.join(self.folder, "finished"))
self.finishedFolder = None
self.status = "initialized"
for solver in self.solverSetups:
solver.refreshLockdown()
self.runner.refreshLockdown()
self.write()
def addAnalysis(self, type, name=None):
"Create and add an analysis of the given type and name"
ana = AnalysisInterface.createAndLoadAnalysis(type, self, os.path.join(self.folder, "analysis"), name)
self.analysis[ana.instName] = ana
def delSolverSetup(self, name):
"Delete a solverSetup specified by name"
#Check that the solverSetup exists
idx = 0
for s in self.solverSetups:
if s.name == name:
break #Found it!
idx += 1
if idx == len(self.solverSetups) or len(self.solverSetups) == 0:
raise AcdOptiException_runConfig_solverSetupDelError("No solversetup named '" + name + "' found")
metaSetupFile = self.solverSetups[idx].metaSetupFilePath
del self.solverSetups[idx]
os.remove(metaSetupFile)
self.write()
def write(self):
"""
Updates the __paramFile and writes it to disk.
"""
print "AcdOptiRunConfig::write()"
self.__paramFile.dataDict.setValSingle("status",self.status)
if self.stageName:
self.__paramFile.dataDict.setValSingle("stageName", self.stageName)
# self.__paramFile.dataDict.setValSingle("stageFolder", self.stageFolder)
# self.__paramFile.dataDict.setValSingle("stageFile", self.stageFile)
else:
self.__paramFile.dataDict.setValSingle("stageName", "")
# self.__paramFile.dataDict.setValSingle("stageFolder", "")
# self.__paramFile.dataDict.setValSingle("stageFile", "")
# if self.finishedFolder != None:
# assert self.status == "finished"
# self.__paramFile.dataDict.setValSingle("finishedFolder", self.finishedFolder)
# else:
# assert self.status != "finished"
# self.__paramFile.dataDict.setValSingle("finishedFolder", "")
#Solver setups
self.__paramFile.dataDict.delItem("solverSetup")
for ss in self.solverSetups:
self.__paramFile.dataDict.pushBack("solverSetup", ss.name)
#Runner
self.__paramFile.dataDict.setValSingle("runnerType", self.runner.type)
#Analysis
anaDict = self.__paramFile.dataDict["analysis"]
anaDict.clear()
for (key, val) in self.analysis.iteritems():
anaDict.pushBack(key,val.generateRunConfigDict())
self.__paramFile.write()
def __del__(self):
self.write()
@staticmethod
def createNew(folder, runnerType, solverTypes=None):
"""
Create a new RunConfig, if wanted with ready-made
SolverSetups and Runners created and attached.
Input:
- Folder : In which folder to store the setup files etc.
- runnerType : String with the runner type wanted (example: Hopper)
- solverTypes : String or list of strings with the types of SolverSetups wanted
(as accepted by AcdOptiSolverSetup.createNew()).
If None, don't attach any solvers
"""
print "AcdOptiRunConfig::createNew(), folder=" + folder
#Construct the instance name from folder
instname = folder
if instname[-1] == "/":
instname = instname[0:-1]
instname = os.path.split(instname)[1]
if os.path.isdir(folder):
raise AcdOptiException_runConfig_createFail ("Folder \"" + folder + "\" already exists")
os.mkdir(folder)
#Create the SolverSetups:
solverSetups = []
if solverTypes == None:
pass #To simplify the "else" at the end...
elif type(solverTypes) == str:
solverSetups.append(AcdOptiSolverSetup.createNew(solverTypes, folder))
elif type(solverTypes) == list:
for st in solverTypes:
solverSetups.append(AcdOptiSolverSetup.createNew(st, folder))
else:
raise AcdOptiException_runConfig_createFail ("Expected solverTypes to be a string or list of strings, got " + str(solverTypes))
#Create the runner
AcdOptiRunner.createNew(runnerType, folder)
#Create the paramFile
paramFile = AcdOptiFileParser_simple(\
os.path.join(folder, "paramFile.set"), 'w')
paramFile.dataDict.pushBack("fileID", "runConfigParamFile")
paramFile.dataDict.pushBack("instName", instname)
if len(solverSetups):
paramFile.dataDict.pushBack("status", "initialized")
else:
paramFile.dataDict.pushBack("status", "not_initialized")
paramFile.dataDict.pushBack("stageName", "")
# paramFile.dataDict.pushBack("stageFolder", "")
# paramFile.dataDict.pushBack("stageFile", "")
# paramFile.dataDict.pushBack("finishedFolder", "")
#Pushback all solverSetups under the same key
for ssName in solverSetups:
paramFile.dataDict.pushBack("solverSetup", ssName)
#Runner type
paramFile.dataDict.pushBack("runnerType", runnerType)
#Analysis dictionary
paramFile.dataDict.pushBack("analysis", DataDict())
paramFile.write()
#Create the staging folder
os.mkdir(os.path.join(folder, "stage"))
#Create the finished folder
os.mkdir(os.path.join(folder, "finished"))
#Create the analysis folder
os.mkdir(os.path.join(folder,"analysis"))
@staticmethod
def createNew_clone(folder, cloneFrom, newMeshInstance):
"""
Creates a new runConfig in a not previously existing folder,
which has identical settings as an already existing runConfig.
The newly created runConfig is then returned.
This is a deep copy, runners, solvers, and analysis etc. are also cloned.
"""
#Create the new runConfig
AcdOptiRunConfig.createNew(folder, cloneFrom.runner.type, None)
newRC = AcdOptiRunConfig(folder, newMeshInstance)
#Copy runner data
newRC.runner.cloneInto(cloneFrom.runner)
#Copy the solvers
for solv in cloneFrom.solverSetups:
print "Add solver " + str(solv)
newSolv = AcdOptiSolverSetup.createNew_clone(folder, solv,newRC)
newRC.solverSetups.append(newSolv)
#Copy analysis
for (anaName, ana) in cloneFrom.analysis.iteritems():
newRC.analysis[anaName] = ana.createNew_clone(os.path.join(folder, "analysis"), ana, newRC)
newRC.refreshStatus()
newRC.write()
return newRC
