def _plot(self, gridFile, solutionFile=None, aggregatesFile=None, connectionsFile=None):
config = DOUGConfigParser(name='Plot parameters')
config.addConfig(self.app.config)
configPanel = ConfigPanel(self.app.root, config, title="Plot configuration",
sectionNames=['gridplot'])
if not configPanel.done:
return
LOG.info("Show plot of %s", gridFile)
gridplot=os.path.join(os.path.dirname(sys.argv[0]), 'gridplot.py')
args = [gridplot, gridplot]
args.extend(['--gin', gridFile])
if solutionFile:
args.extend(['--sin', solutionFile])
if aggregatesFile:
args.extend(['--ain', aggregatesFile])
if connectionsFile:
args.extend(['--cin', connectionsFile])
plargs=[]
device=config.get('gridplot', 'device')
plargs.extend(['-dev', device])
aggr=config.get('gridplot', 'aggr', None)
if aggr:
args.extend(['--aggr', aggr])
args.extend(['--plplot', " ".join(plargs)])
LOG.debug("Spawn '%s'", args)
os.spawnvp(os.P_NOWAIT, 'python', args)
def createTestRun():
gconfFile = tarFile.extractfile(testtar.GLOBAL_CONF)
gconfContent = gconfFile.read()
config = DOUGConfigParser()
config.addConfigContents(gconfContent)
startTime = float(config.get("testrun", "starttime"))
stopTime = float(config.get("testrun", "stoptime"))
store.insertTestRun(config, datetime.fromtimestamp(startTime))
store.updateTestRunStop(datetime.fromtimestamp(stopTime))
def __init__(self, config, dougControls=None):
self.workdir = os.path.abspath(config.get('doug', 'workdir'))
self.config = DOUGConfigParser(name='DOUG execution', basedir=self.workdir)
# default config
self.config.addConfig(getDefaultConfig())
self.config.addControlFile(getDefaultControlFile(self.workdir))
# copy controls from control file
if dougControls is not None:
self.config.addControlFile(dougControls)
# copy config
self.config.addConfig(config)
# output or other files, exception grabs it on exit
self.files = []
# how many test results are using this test, files are deleted only after last free() call
self._inUse = 0
self.preserveOutput = self.config.getboolean("doug", "preserveOutput")
self.result = DOUGConfigParser(self.config.defaults(), basedir=self.workdir)
self.result.add_section('doug-result')
def __init__(self, name=None, directoryName=None, archiveType='problem'):
self.name = name
self.filetypes = {} #: file name -> type
if directoryName==None:
directoryName = os.path.abspath(name)
if os.path.isdir(directoryName):
self._state = Archive.SAVED
self.directoryName = directoryName
self.load()
else:
os.mkdir(directoryName)
self._state = Archive.DIRTY
self.directoryName = directoryName
self.info = DOUGConfigParser()
self.info.add_section('general')
self.info.set('general', 'archive-type', archiveType)
self.info.add_section('files')
def createTest(testdir, members):
del members[testtar.TEST_CONF]
testConfigFile = tarFile.extractfile("/".join([testdir.name, testtar.TEST_CONF]))
testConfigContent = testConfigFile.read()
del members[testtar.RESULT_CONF]
testResultFile = tarFile.extractfile("/".join([testdir.name, testtar.RESULT_CONF]))
testResultContent = testResultFile.read()
config = DOUGConfigParser()
config.addConfigContents(testConfigContent)
config.addConfigContents(testResultContent)
# store test times and status
testname = config.get("test", "name")
startTime = float(config.get("test", "starttime"))
stopTime = float(config.get("test", "stoptime"))
ID = store.insertTest(testname, datetime.fromtimestamp(startTime), config)
store.updateTestStop(ID, datetime.fromtimestamp(stopTime))
status = config.get("doug-result", "status").upper()
status = testmysql.TestStatus.STRS.index(status)
store.updateTestStatus(ID, status)
# store error
if members.has_key(testtar.EXCEPTION_FILE):
del members[testtar.EXCEPTION_FILE]
exceptionFile = tarFile.extractfile("/".join([testdir.name, testtar.EXCEPTION_FILE]))
exc = pickle.load(exceptionFile)
store.updateTestError(ID, str(exc))
# store profile info
if config.has_section('doug-profile'):
store.updateTestProfileInfo(ID, config)
# left files should be stored to database
for name in members.keys():
file = tarFile.extractfile("/".join([testdir.name, name]))
fileContent = file.read()
store.insertTestFileContent(ID, name, fileContent)
opts, extra = getopt.getopt(sys.argv[1:], "", ["conf="])
except getopt.GetoptError:
usage()
sys.exit(1)
if len(extra)>0:
usage()
sys.exit(1)
confFileNames = []
for opt in opts:
if opt[0]=="--conf":
confFileNames.append(opt[1])
# config and tar file
conf = DOUGConfigParser()
conf.read(confFileNames)
host = conf.get("testscript", "mysql-host")
user = conf.get("testscript", "mysql-user")
password = conf.get("testscript", "mysql-password")
database = conf.get("testscript", "mysql-database")
class TestResult:
def __init__(self, **kvargs):
self.__dict__.update(kvargs)
def __str__(self):
return "TestResult(ID=%d)"%self.ID
def getKeyHeader(self):
class Archive(object):
DIRTY = 'DIRTY'
SAVED = 'SAVED'
ARCHIVED = 'ARCHIVED'
FILETYPES = ['', 'Grid', 'Matrix', 'Vector', 'Vector/RHS', 'Vector/Solution',
'Aggregates/Fine', 'Aggregates/Coarse', 'Text', 'Text/Output', 'Text/Error',
'Matrix/Connections']
def _setState(self, newstate): self._state=newstate
state = property(lambda self: self._state, fset=_setState)
def __init__(self, name=None, directoryName=None, archiveType='problem'):
self.name = name
self.filetypes = {} #: file name -> type
if directoryName==None:
directoryName = os.path.abspath(name)
if os.path.isdir(directoryName):
self._state = Archive.SAVED
self.directoryName = directoryName
self.load()
else:
os.mkdir(directoryName)
self._state = Archive.DIRTY
self.directoryName = directoryName
self.info = DOUGConfigParser()
self.info.add_section('general')
self.info.set('general', 'archive-type', archiveType)
self.info.add_section('files')
def save(self):
if self.state == Archive.DIRTY:
LOG.info("Saving '%s' to %s" % (self.name, self.directoryName))
config = copy.deepcopy(self.info)
config.set('general', 'name', self.name)
configFile = file(os.path.join(self.directoryName, '.info'), 'w')
files=map(lambda f: "%s:%s"%f, self.filetypes.items())
config.set('files', 'types', ", ".join(files))
config.write(configFile)
configFile.close()
def load(self):
LOG.info("Loading from %s" % self.directoryName)
config = DOUGConfigParser()
config.read([os.path.join(self.directoryName, '.info')])
self.info = config
if config.has_section('general'):
self.name = config.get('general', 'name')
else:
config.add_section('general')
if config.has_section('files'):
types = config.get('files', 'types', default="")
types = types.split(",")
types = filter(bool, types) # filter out empty strings
types = map(lambda s: map(str.strip, s.split(':')), types)
for filename, filetype in types:
self.setFileType(filename, filetype)
else:
config.add_section('files')
def close(self):
self.save()
def setFileType(self, filename, filetype):
if filetype==None:
if self.filetypes.has_key(filename):
del self.filetypes[filename]
else:
self.filetypes[filename] = filetype
if filetype=='Text/Profile':
self._readProfileFile(filename)
self.state = Archive.DIRTY
def _readProfileFile(self, filename):
f = open(os.path.join(self.directoryName, filename))
try:
for line in f:
line = line.strip()
proc, name, value = line.split(':')
name = name.replace(' ', '-')
self.info.set('doug-profile', name, value, addsection=True)
finally:
f.close()
def getFileType(self, filename):
return self.filetypes.get(filename, None)
def getFiles(self, filetype):
files = map(lambda g: g[0],
filter(lambda x: x[1]==filetype,
self.filetypes.items()))
return files
def __str__(self):
return "Archive(directory=%s)" % self.directoryName
def load(self):
LOG.info("Loading from %s" % self.directoryName)
config = DOUGConfigParser()
config.read([os.path.join(self.directoryName, '.info')])
self.info = config
if config.has_section('general'):
self.name = config.get('general', 'name')
else:
config.add_section('general')
if config.has_section('files'):
types = config.get('files', 'types', default="")
types = types.split(",")
types = filter(bool, types) # filter out empty strings
types = map(lambda s: map(str.strip, s.split(':')), types)
for filename, filetype in types:
self.setFileType(filename, filetype)
else:
config.add_section('files')
class DOUGExecution:
def __init__(self, config, dougControls=None):
self.workdir = os.path.abspath(config.get('doug', 'workdir'))
self.config = DOUGConfigParser(name='DOUG execution', basedir=self.workdir)
# default config
self.config.addConfig(getDefaultConfig())
self.config.addControlFile(getDefaultControlFile(self.workdir))
# copy controls from control file
if dougControls is not None:
self.config.addControlFile(dougControls)
# copy config
self.config.addConfig(config)
# output or other files, exception grabs it on exit
self.files = []
# how many test results are using this test, files are deleted only after last free() call
self._inUse = 0
self.preserveOutput = self.config.getboolean("doug", "preserveOutput")
self.result = DOUGConfigParser(self.config.defaults(), basedir=self.workdir)
self.result.add_section('doug-result')
def setUp(self):
LOG.debug("Preparing testing environment")
self.workdir = self.workdir
if os.path.isdir(self.workdir):
self.workdirExisted = True
else:
self.workdirExisted = False
os.mkdir(self.workdir)
LOG.debug("Working directory %s created" % self.workdir)
try:
# create control file
self.testctrlfname = os.path.abspath(os.path.join(self.workdir, 'DOUG-exec.ctl'))
controlFile = self.config.getControlFile(self.testctrlfname)
controlFile.save(self.testctrlfname)
self.files.append((self.testctrlfname, "Control file"))
# run mpiboot
mpibootname = self.config.get("doug", "mpiboot")
outfilename = self.config.get("doug", "mpiboot-outfilename")
errfilename = self.config.get("doug", "mpiboot-errfilename")
if mpibootname:
LOG.debug("Setting up mpi")
mpiboot = subprocess.Popen("%s > %s 2> %s" % (mpibootname, outfilename, errfilename), shell=True)
res = mpiboot.wait()
if res:
raise ScriptException("Error running %s (%d)"
"inspect output files (%s, %s) for error description."
% (mpibootname, res, outfilename, errfilename))
except:
self._clean()
raise
def tearDown(self):
try:
mpihaltname = self.config.get("doug", "mpihalt")
if mpihaltname:
outfilename = self.config.get("doug", "mpihalt-outfilename")
errfilename = self.config.get("doug", "mpihalt-errfilename")
LOG.debug("Shutting down mpi")
mpihalt = subprocess.Popen("%s > %s 2> %s" % (mpihaltname, outfilename, errfilename), shell=True)
import time
time.sleep(4) # lamhalt <=7.1.1 does not wait until whole universe is shut down
res = mpihalt.wait()
if res:
LOG.warn("Error running %s (%d)"
"inspect output files (%s, %s) for error description."
% (mpihaltname, res, outfilename, errfilename))
except Exception, e:
LOG.warn("Exception running mpihalt: %s" % e)
def getDefaultConfig():
global _defaultConfig
if _defaultConfig is None:
_defaultConfig = DOUGConfigParser(name="DOUG default config")
_defaultConfig.addConfigContents(doug.execution_conf_tmpl)
return _defaultConfig
def _solve(self, initialConfig, problemArchive):
try:
config = DOUGConfigParser(name='DOUG execution parameters')
config.addConfigContents(doug.execution.getDefaultConfigContents())
config.addConfig(initialConfig)
suffix = time.strftime('--run-%y%m%d-%H%M%S')
config.set('DEFAULT', 'doug-workdir', problemArchive.directoryName+suffix)
config.set('DEFAULT', 'doug-datadir', problemArchive.directoryName)
matrixFiles = problemArchive.getFiles('Matrix')
if matrixFiles:
matrixFile='%(doug-datadir)s/'+matrixFiles[0]
else:
matrixFile=''
config.set('doug-controls', 'assembled_mtx_file', matrixFile)
rhsFiles = problemArchive.getFiles('Vector/RHS')
if rhsFiles:
rhsFile='%(doug-datadir)s/'+rhsFiles[0]
else:
rhsFile=''
config.set('doug-controls', 'assembled_rhs_file', rhsFile)
# change configurations
configPanel = ConfigPanel(self.frame, config, title="DOUG configuration",
sectionNames=['DEFAULT', 'doug', 'doug-controls'])
if not configPanel.done:
return
# run
execution = doug.execution.DOUGExecution(config)
try:
archive = Archive(problemArchive.name+suffix,
directoryName=problemArchive.directoryName+suffix,
archiveType='solution')
archive.info.set('general','problem-name',problemArchive.name)
archive.info.addConfig(execution.config)
archive.info.set('doug-execution', 'date', time.strftime('%y-%m-%d %H:%M:%S'), addsection=True)
resultConfig=execution.run()
archive.info.addConfig(resultConfig)
if archive.info.has_option('doug-result', 'solutionfile'):
archive.setFileType(archive.info.get('doug-result', 'solutionfile'), 'Vector/Solution')
if archive.info.has_option('doug-result', 'errorfile'):
archive.setFileType(archive.info.get('doug-result', 'errorfile'), 'Text/Error')
if archive.info.has_option('doug-result', 'outputfile'):
archive.setFileType(archive.info.get('doug-result', 'outputfile'), 'Text/Output')
if archive.info.has_option('doug-result', 'fineaggrsfile'):
archive.setFileType(archive.info.get('doug-result', 'fineaggrsfile'), 'Aggregates/Fine')
if archive.info.has_option('doug-result', 'coarseaggrsfile'):
archive.setFileType(archive.info.get('doug-result', 'coarseaggrsfile'), 'Aggregates/Coarse')
if archive.info.has_option('doug-result', 'profilefile'):
archive.setFileType(archive.info.get('doug-result', 'profilefile'), 'Text/Profile')
finally:
self.app.addArchive(archive)
except(Exception), e:
LOG.error(e, exc_info=e)
class App:
def __reload(self, ev):
import doug.execution
print "Reloading execution"
reload(doug.execution)
def __init__(self, root):
self.root = root
root.title('DOUG execution manager')
root.geometry('800x600')
self.problemsArchives = []
self.solutionsArchives = []
self.__loadOptions()
root.protocol('WM_DELETE_WINDOW', self.__closeApp)
root.bind('<Control-R>', self.__reload)
bottomFrame = Frame(root)
bottomFrame.pack(side=BOTTOM, fill=X)
menubar = Menu(root)
root.config(menu=menubar)
# main frame
mainFrame=Pmw.NoteBook(root)
self.notebook=mainFrame
mainFrame.pack(expand=True, fill=BOTH)
problemsFrame = LabelFrame(mainFrame.add('problems'), text='Problems', fg='blue')
problemsFrame.pack(side=TOP, expand=True, fill=BOTH)
solutionsFrame = LabelFrame(mainFrame.add('solutions'), text='Solutions', fg='blue')
solutionsFrame.pack(side=TOP, expand=True, fill=BOTH)
resultsFrame = LabelFrame(mainFrame.add('results'), text='Results', fg='blue')
resultsFrame.pack(side=TOP, expand=True, fill=BOTH)
# problems listbox
self.problemPanel = ProblemArchivePanel(problemsFrame, self.config, self)
listbox = ArchiveListbox(problemsFrame,
archiveList=self.problemsArchives,
archivePanel=self.problemPanel)
self.problemsListbox = listbox
listbox.pack(side=LEFT, fill=Y)
self.problemPanel.frame.pack(side=TOP, expand=True, fill=BOTH)
addb = Button(problemsFrame, text="Add archive", command=self.__addArchive)
addb.pack(side=TOP)
# solutions listbox
self.solutionPanel = SolutionArchivePanel(solutionsFrame, self.config, self)
listbox = ArchiveListbox(solutionsFrame,
archiveList=self.solutionsArchives,
archivePanel=self.solutionPanel)
self.solutionsListbox = listbox
listbox.pack(side=LEFT, fill=Y)
self.solutionPanel.frame.pack(side=TOP, expand=True, fill=BOTH)
menu = Menu(menubar, tearoff=False)
menu.add_command(label='Quit', command=self.__closeApp)
menubar.add_cascade(label='File', menu=menu)
menu = Menu(menubar, tearoff=False)
menu.add_command(label='Options', command=self.__editOptions)
menubar.add_cascade(label='Tools', menu=menu)
self.__scanDirectory('.', {
'problem': (self.problemsArchives, self.problemsListbox),
'solution': (self.solutionsArchives, self.solutionsListbox)})
# results frame
resultTable = ResultTable(self, resultsFrame)
refreshB = Button(resultsFrame, text='refresh', fg="green", command=resultTable.refresh)
refreshB.pack(anchor=W)
resultTable.frame.pack(fill=BOTH, expand=True)
def __editOptions(self):
d = ConfigPanel(self.root, self.config, title='Global options')
if d.done:
self.__saveOptions()
def __scanDirectory(self, dirpath, archiveTypes):
dirs = os.listdir(dirpath)
dirs = map(lambda x: os.path.join(dirpath, x), dirs) # create full paths
dirs = filter(os.path.isdir, dirs) # filter directories
for subdir in dirs:
if os.path.exists(os.path.join(subdir, '.info')):
abssubdir = os.path.abspath(subdir)
archive = Archive(directoryName = abssubdir)
t = archive.info.get('general', 'archive-type')
if t in archiveTypes:
archiveTypes[t][0].append(archive)
archiveTypes[t][1].insert(END, archive.name)
def addArchive(self, archive):
archiveType = archive.info.get('general', 'archive-type')
if archiveType=='problem':
self.problemsArchives.append(archive)
self.problemsListbox.insert(END, archive.name)
self.notebook.selectpage('problems')
self.problemsListbox.focus()
elif archiveType=='solution':
self.solutionsArchives.append(archive)
self.notebook.selectpage('solutions')
self.solutionsListbox.focus()
self.solutionsListbox.insert(END, archive.name)
self.solutionsListbox.setvalue(archive.name)
def __addArchive(self):
name = tkSimpleDialog.askstring("Insert name", "Type new archive name (without .tar.gz)")
if name:
archive = Archive(name)
self.addArchive(archive)
def __closeApp(self):
for archive in self.problemsArchives:
archive.close()
for archive in self.solutionsArchives:
archive.close()
LOG.info("Closing application")
self.root.destroy()
def __loadOptions(self):
self.config = DOUGConfigParser()
self.config.addConfigContents(_configContent)
if os.path.exists('.manager.conf'):
self.config.read('.manager.conf')
def __saveOptions(self):
f = open('.manager.conf', 'w')
try:
LOG.info('Saving %s' % f)
self.config.write(f)
finally:
f.close()
def getProblemArchive(self, name):
for archive in self.problemsArchives:
if archive.name==name:
return archive
return None
def __loadOptions(self):
self.config = DOUGConfigParser()
self.config.addConfigContents(_configContent)
if os.path.exists('.manager.conf'):
self.config.read('.manager.conf')
def main(testResults):
# svn
if conf.getboolean('testscript', 'run-svn'):
svndoug = svnscripts.run(confFileNames, conf.defaults())
# autotools
if conf.getboolean('testscript', 'run-autotools'):
autotools.run(confFileNames, conf.defaults())
# construct tests
if conf.getboolean('testscript', 'run-tests'):
testSuite = unittest.TestSuite()
items = conf.items("tests")
for name, value in items:
if not name.startswith("test"): continue
tname = name[4:]
# read test data
ctrlfname, solutionfname, testconfs = tuple(value.strip().split(" ", 2))
ctrlfname = os.path.abspath(ctrlfname)
solutionfname = os.path.abspath(solutionfname)
datadir = os.path.dirname(ctrlfname)
LOG.debug("Constructing test '%s'" % (tname, ))
LOG.debug("Control file: %s" % ctrlfname)
LOG.debug("Correct solution file: %s" % (solutionfname,))
# read test configurations
testconfs = testconfs.split(",")
for testconf in testconfs:
testconfname = "testconf_%s" % testconf
testname = "%s_%s" % (tname, testconf)
solvers = map(int, conf.get(testconfname, "solver").split(","))
methods = map(int, conf.get(testconfname, "method").split(","))
levels = map(int, conf.get(testconfname, "levels").split(","))
fine_methods = map(int, conf.get(testconfname, "fine_methods").split(","))
nsubdomains = map(int, conf.get(testconfname, "num_subdomains").split(","))
processors = map(int, conf.get(testconfname, "processors").split(","))
executables = conf.get(testconfname, "executables").split(",")
overlaps = map(int, conf.get(testconfname, "overlaps").split(","))
smoothers = map(int, conf.get(testconfname, "smoothers").split(","))
testtuples = generateTuples(solvers, methods, levels, fine_methods, nsubdomains, processors, executables, overlaps, smoothers)
for solver,method,level,fine_method, nsubdomain,nproc,executable,overlap,smoother in testtuples:
dougControlFile = doug.execution.ControlFile(filename=ctrlfname, basedir=os.path.dirname(ctrlfname))
dougConfig = DOUGConfigParser(name='DOUG execution parameters')
# set/copy doug configuration from tests configuration
dougConfig.add_section('doug')
for name,value in conf.items('doug', raw=True):
dougConfig.set('doug', name, value)
dougConfig.add_section('doug-controls')
for name,value in conf._sections['doug-controls'].items():
dougConfig.set('doug-controls', name, value)
dougConfig.add_section('doug-tests')
dougConfig.set('doug-tests', 'csolution_file', solutionfname)
# set test configuration
dougConfig.set('doug-controls', 'solver', str(solver))
dougConfig.set('doug-controls', 'method', str(method))
dougConfig.set('doug-controls', 'levels', str(level))
dougConfig.set('doug-controls', 'fine_method', str(fine_method))
dougConfig.set('doug-controls', 'num_subdomains', str(nsubdomain))
dougConfig.set('doug', 'nproc', str(nproc))
dougConfig.set('doug', 'executable', executable)
dougConfig.set('doug-controls', 'overlap', str(overlap))
dougConfig.set('doug-controls', 'smoothers', str(smoother))
# create DOUG execution object
execution = doug.execution.DOUGExecution(dougConfig, dougControlFile)
test = dougtest.TestCase(execution, testname)
#resultConfig=execution.run()
#test = dougtest.MPITestCase(name+"_"+testconf, datadir, ctrlfname, solutionfname, conf, *testtuple)
testSuite.addTest(test)
# run tests
testRunner = dougtest.TestRunner(testResults)
testRunner.run(testSuite)
