def _plot(self, gridFile, solutionFile=None, aggregatesFile=None, connectionsFile=None): config = DOUGConfigParser(name='Plot parameters') config.addConfig(self.app.config) configPanel = ConfigPanel(self.app.root, config, title="Plot configuration", sectionNames=['gridplot']) if not configPanel.done: return LOG.info("Show plot of %s", gridFile) gridplot=os.path.join(os.path.dirname(sys.argv[0]), 'gridplot.py') args = [gridplot, gridplot] args.extend(['--gin', gridFile]) if solutionFile: args.extend(['--sin', solutionFile]) if aggregatesFile: args.extend(['--ain', aggregatesFile]) if connectionsFile: args.extend(['--cin', connectionsFile]) plargs=[] device=config.get('gridplot', 'device') plargs.extend(['-dev', device]) aggr=config.get('gridplot', 'aggr', None) if aggr: args.extend(['--aggr', aggr]) args.extend(['--plplot', " ".join(plargs)]) LOG.debug("Spawn '%s'", args) os.spawnvp(os.P_NOWAIT, 'python', args)
def createTestRun(): gconfFile = tarFile.extractfile(testtar.GLOBAL_CONF) gconfContent = gconfFile.read() config = DOUGConfigParser() config.addConfigContents(gconfContent) startTime = float(config.get("testrun", "starttime")) stopTime = float(config.get("testrun", "stoptime")) store.insertTestRun(config, datetime.fromtimestamp(startTime)) store.updateTestRunStop(datetime.fromtimestamp(stopTime))
def __init__(self, config, dougControls=None): self.workdir = os.path.abspath(config.get('doug', 'workdir')) self.config = DOUGConfigParser(name='DOUG execution', basedir=self.workdir) # default config self.config.addConfig(getDefaultConfig()) self.config.addControlFile(getDefaultControlFile(self.workdir)) # copy controls from control file if dougControls is not None: self.config.addControlFile(dougControls) # copy config self.config.addConfig(config) # output or other files, exception grabs it on exit self.files = [] # how many test results are using this test, files are deleted only after last free() call self._inUse = 0 self.preserveOutput = self.config.getboolean("doug", "preserveOutput") self.result = DOUGConfigParser(self.config.defaults(), basedir=self.workdir) self.result.add_section('doug-result')
def __init__(self, name=None, directoryName=None, archiveType='problem'): self.name = name self.filetypes = {} #: file name -> type if directoryName==None: directoryName = os.path.abspath(name) if os.path.isdir(directoryName): self._state = Archive.SAVED self.directoryName = directoryName self.load() else: os.mkdir(directoryName) self._state = Archive.DIRTY self.directoryName = directoryName self.info = DOUGConfigParser() self.info.add_section('general') self.info.set('general', 'archive-type', archiveType) self.info.add_section('files')
def createTest(testdir, members): del members[testtar.TEST_CONF] testConfigFile = tarFile.extractfile("/".join([testdir.name, testtar.TEST_CONF])) testConfigContent = testConfigFile.read() del members[testtar.RESULT_CONF] testResultFile = tarFile.extractfile("/".join([testdir.name, testtar.RESULT_CONF])) testResultContent = testResultFile.read() config = DOUGConfigParser() config.addConfigContents(testConfigContent) config.addConfigContents(testResultContent) # store test times and status testname = config.get("test", "name") startTime = float(config.get("test", "starttime")) stopTime = float(config.get("test", "stoptime")) ID = store.insertTest(testname, datetime.fromtimestamp(startTime), config) store.updateTestStop(ID, datetime.fromtimestamp(stopTime)) status = config.get("doug-result", "status").upper() status = testmysql.TestStatus.STRS.index(status) store.updateTestStatus(ID, status) # store error if members.has_key(testtar.EXCEPTION_FILE): del members[testtar.EXCEPTION_FILE] exceptionFile = tarFile.extractfile("/".join([testdir.name, testtar.EXCEPTION_FILE])) exc = pickle.load(exceptionFile) store.updateTestError(ID, str(exc)) # store profile info if config.has_section('doug-profile'): store.updateTestProfileInfo(ID, config) # left files should be stored to database for name in members.keys(): file = tarFile.extractfile("/".join([testdir.name, name])) fileContent = file.read() store.insertTestFileContent(ID, name, fileContent)
opts, extra = getopt.getopt(sys.argv[1:], "", ["conf="]) except getopt.GetoptError: usage() sys.exit(1) if len(extra)>0: usage() sys.exit(1) confFileNames = [] for opt in opts: if opt[0]=="--conf": confFileNames.append(opt[1]) # config and tar file conf = DOUGConfigParser() conf.read(confFileNames) host = conf.get("testscript", "mysql-host") user = conf.get("testscript", "mysql-user") password = conf.get("testscript", "mysql-password") database = conf.get("testscript", "mysql-database") class TestResult: def __init__(self, **kvargs): self.__dict__.update(kvargs) def __str__(self): return "TestResult(ID=%d)"%self.ID def getKeyHeader(self):
class Archive(object): DIRTY = 'DIRTY' SAVED = 'SAVED' ARCHIVED = 'ARCHIVED' FILETYPES = ['', 'Grid', 'Matrix', 'Vector', 'Vector/RHS', 'Vector/Solution', 'Aggregates/Fine', 'Aggregates/Coarse', 'Text', 'Text/Output', 'Text/Error', 'Matrix/Connections'] def _setState(self, newstate): self._state=newstate state = property(lambda self: self._state, fset=_setState) def __init__(self, name=None, directoryName=None, archiveType='problem'): self.name = name self.filetypes = {} #: file name -> type if directoryName==None: directoryName = os.path.abspath(name) if os.path.isdir(directoryName): self._state = Archive.SAVED self.directoryName = directoryName self.load() else: os.mkdir(directoryName) self._state = Archive.DIRTY self.directoryName = directoryName self.info = DOUGConfigParser() self.info.add_section('general') self.info.set('general', 'archive-type', archiveType) self.info.add_section('files') def save(self): if self.state == Archive.DIRTY: LOG.info("Saving '%s' to %s" % (self.name, self.directoryName)) config = copy.deepcopy(self.info) config.set('general', 'name', self.name) configFile = file(os.path.join(self.directoryName, '.info'), 'w') files=map(lambda f: "%s:%s"%f, self.filetypes.items()) config.set('files', 'types', ", ".join(files)) config.write(configFile) configFile.close() def load(self): LOG.info("Loading from %s" % self.directoryName) config = DOUGConfigParser() config.read([os.path.join(self.directoryName, '.info')]) self.info = config if config.has_section('general'): self.name = config.get('general', 'name') else: config.add_section('general') if config.has_section('files'): types = config.get('files', 'types', default="") types = types.split(",") types = filter(bool, types) # filter out empty strings types = map(lambda s: map(str.strip, s.split(':')), types) for filename, filetype in types: self.setFileType(filename, filetype) else: config.add_section('files') def close(self): self.save() def setFileType(self, filename, filetype): if filetype==None: if self.filetypes.has_key(filename): del self.filetypes[filename] else: self.filetypes[filename] = filetype if filetype=='Text/Profile': self._readProfileFile(filename) self.state = Archive.DIRTY def _readProfileFile(self, filename): f = open(os.path.join(self.directoryName, filename)) try: for line in f: line = line.strip() proc, name, value = line.split(':') name = name.replace(' ', '-') self.info.set('doug-profile', name, value, addsection=True) finally: f.close() def getFileType(self, filename): return self.filetypes.get(filename, None) def getFiles(self, filetype): files = map(lambda g: g[0], filter(lambda x: x[1]==filetype, self.filetypes.items())) return files def __str__(self): return "Archive(directory=%s)" % self.directoryName
def load(self): LOG.info("Loading from %s" % self.directoryName) config = DOUGConfigParser() config.read([os.path.join(self.directoryName, '.info')]) self.info = config if config.has_section('general'): self.name = config.get('general', 'name') else: config.add_section('general') if config.has_section('files'): types = config.get('files', 'types', default="") types = types.split(",") types = filter(bool, types) # filter out empty strings types = map(lambda s: map(str.strip, s.split(':')), types) for filename, filetype in types: self.setFileType(filename, filetype) else: config.add_section('files')
class DOUGExecution: def __init__(self, config, dougControls=None): self.workdir = os.path.abspath(config.get('doug', 'workdir')) self.config = DOUGConfigParser(name='DOUG execution', basedir=self.workdir) # default config self.config.addConfig(getDefaultConfig()) self.config.addControlFile(getDefaultControlFile(self.workdir)) # copy controls from control file if dougControls is not None: self.config.addControlFile(dougControls) # copy config self.config.addConfig(config) # output or other files, exception grabs it on exit self.files = [] # how many test results are using this test, files are deleted only after last free() call self._inUse = 0 self.preserveOutput = self.config.getboolean("doug", "preserveOutput") self.result = DOUGConfigParser(self.config.defaults(), basedir=self.workdir) self.result.add_section('doug-result') def setUp(self): LOG.debug("Preparing testing environment") self.workdir = self.workdir if os.path.isdir(self.workdir): self.workdirExisted = True else: self.workdirExisted = False os.mkdir(self.workdir) LOG.debug("Working directory %s created" % self.workdir) try: # create control file self.testctrlfname = os.path.abspath(os.path.join(self.workdir, 'DOUG-exec.ctl')) controlFile = self.config.getControlFile(self.testctrlfname) controlFile.save(self.testctrlfname) self.files.append((self.testctrlfname, "Control file")) # run mpiboot mpibootname = self.config.get("doug", "mpiboot") outfilename = self.config.get("doug", "mpiboot-outfilename") errfilename = self.config.get("doug", "mpiboot-errfilename") if mpibootname: LOG.debug("Setting up mpi") mpiboot = subprocess.Popen("%s > %s 2> %s" % (mpibootname, outfilename, errfilename), shell=True) res = mpiboot.wait() if res: raise ScriptException("Error running %s (%d)" "inspect output files (%s, %s) for error description." % (mpibootname, res, outfilename, errfilename)) except: self._clean() raise def tearDown(self): try: mpihaltname = self.config.get("doug", "mpihalt") if mpihaltname: outfilename = self.config.get("doug", "mpihalt-outfilename") errfilename = self.config.get("doug", "mpihalt-errfilename") LOG.debug("Shutting down mpi") mpihalt = subprocess.Popen("%s > %s 2> %s" % (mpihaltname, outfilename, errfilename), shell=True) import time time.sleep(4) # lamhalt <=7.1.1 does not wait until whole universe is shut down res = mpihalt.wait() if res: LOG.warn("Error running %s (%d)" "inspect output files (%s, %s) for error description." % (mpihaltname, res, outfilename, errfilename)) except Exception, e: LOG.warn("Exception running mpihalt: %s" % e)
def getDefaultConfig(): global _defaultConfig if _defaultConfig is None: _defaultConfig = DOUGConfigParser(name="DOUG default config") _defaultConfig.addConfigContents(doug.execution_conf_tmpl) return _defaultConfig
def _solve(self, initialConfig, problemArchive): try: config = DOUGConfigParser(name='DOUG execution parameters') config.addConfigContents(doug.execution.getDefaultConfigContents()) config.addConfig(initialConfig) suffix = time.strftime('--run-%y%m%d-%H%M%S') config.set('DEFAULT', 'doug-workdir', problemArchive.directoryName+suffix) config.set('DEFAULT', 'doug-datadir', problemArchive.directoryName) matrixFiles = problemArchive.getFiles('Matrix') if matrixFiles: matrixFile='%(doug-datadir)s/'+matrixFiles[0] else: matrixFile='' config.set('doug-controls', 'assembled_mtx_file', matrixFile) rhsFiles = problemArchive.getFiles('Vector/RHS') if rhsFiles: rhsFile='%(doug-datadir)s/'+rhsFiles[0] else: rhsFile='' config.set('doug-controls', 'assembled_rhs_file', rhsFile) # change configurations configPanel = ConfigPanel(self.frame, config, title="DOUG configuration", sectionNames=['DEFAULT', 'doug', 'doug-controls']) if not configPanel.done: return # run execution = doug.execution.DOUGExecution(config) try: archive = Archive(problemArchive.name+suffix, directoryName=problemArchive.directoryName+suffix, archiveType='solution') archive.info.set('general','problem-name',problemArchive.name) archive.info.addConfig(execution.config) archive.info.set('doug-execution', 'date', time.strftime('%y-%m-%d %H:%M:%S'), addsection=True) resultConfig=execution.run() archive.info.addConfig(resultConfig) if archive.info.has_option('doug-result', 'solutionfile'): archive.setFileType(archive.info.get('doug-result', 'solutionfile'), 'Vector/Solution') if archive.info.has_option('doug-result', 'errorfile'): archive.setFileType(archive.info.get('doug-result', 'errorfile'), 'Text/Error') if archive.info.has_option('doug-result', 'outputfile'): archive.setFileType(archive.info.get('doug-result', 'outputfile'), 'Text/Output') if archive.info.has_option('doug-result', 'fineaggrsfile'): archive.setFileType(archive.info.get('doug-result', 'fineaggrsfile'), 'Aggregates/Fine') if archive.info.has_option('doug-result', 'coarseaggrsfile'): archive.setFileType(archive.info.get('doug-result', 'coarseaggrsfile'), 'Aggregates/Coarse') if archive.info.has_option('doug-result', 'profilefile'): archive.setFileType(archive.info.get('doug-result', 'profilefile'), 'Text/Profile') finally: self.app.addArchive(archive) except(Exception), e: LOG.error(e, exc_info=e)
class App: def __reload(self, ev): import doug.execution print "Reloading execution" reload(doug.execution) def __init__(self, root): self.root = root root.title('DOUG execution manager') root.geometry('800x600') self.problemsArchives = [] self.solutionsArchives = [] self.__loadOptions() root.protocol('WM_DELETE_WINDOW', self.__closeApp) root.bind('<Control-R>', self.__reload) bottomFrame = Frame(root) bottomFrame.pack(side=BOTTOM, fill=X) menubar = Menu(root) root.config(menu=menubar) # main frame mainFrame=Pmw.NoteBook(root) self.notebook=mainFrame mainFrame.pack(expand=True, fill=BOTH) problemsFrame = LabelFrame(mainFrame.add('problems'), text='Problems', fg='blue') problemsFrame.pack(side=TOP, expand=True, fill=BOTH) solutionsFrame = LabelFrame(mainFrame.add('solutions'), text='Solutions', fg='blue') solutionsFrame.pack(side=TOP, expand=True, fill=BOTH) resultsFrame = LabelFrame(mainFrame.add('results'), text='Results', fg='blue') resultsFrame.pack(side=TOP, expand=True, fill=BOTH) # problems listbox self.problemPanel = ProblemArchivePanel(problemsFrame, self.config, self) listbox = ArchiveListbox(problemsFrame, archiveList=self.problemsArchives, archivePanel=self.problemPanel) self.problemsListbox = listbox listbox.pack(side=LEFT, fill=Y) self.problemPanel.frame.pack(side=TOP, expand=True, fill=BOTH) addb = Button(problemsFrame, text="Add archive", command=self.__addArchive) addb.pack(side=TOP) # solutions listbox self.solutionPanel = SolutionArchivePanel(solutionsFrame, self.config, self) listbox = ArchiveListbox(solutionsFrame, archiveList=self.solutionsArchives, archivePanel=self.solutionPanel) self.solutionsListbox = listbox listbox.pack(side=LEFT, fill=Y) self.solutionPanel.frame.pack(side=TOP, expand=True, fill=BOTH) menu = Menu(menubar, tearoff=False) menu.add_command(label='Quit', command=self.__closeApp) menubar.add_cascade(label='File', menu=menu) menu = Menu(menubar, tearoff=False) menu.add_command(label='Options', command=self.__editOptions) menubar.add_cascade(label='Tools', menu=menu) self.__scanDirectory('.', { 'problem': (self.problemsArchives, self.problemsListbox), 'solution': (self.solutionsArchives, self.solutionsListbox)}) # results frame resultTable = ResultTable(self, resultsFrame) refreshB = Button(resultsFrame, text='refresh', fg="green", command=resultTable.refresh) refreshB.pack(anchor=W) resultTable.frame.pack(fill=BOTH, expand=True) def __editOptions(self): d = ConfigPanel(self.root, self.config, title='Global options') if d.done: self.__saveOptions() def __scanDirectory(self, dirpath, archiveTypes): dirs = os.listdir(dirpath) dirs = map(lambda x: os.path.join(dirpath, x), dirs) # create full paths dirs = filter(os.path.isdir, dirs) # filter directories for subdir in dirs: if os.path.exists(os.path.join(subdir, '.info')): abssubdir = os.path.abspath(subdir) archive = Archive(directoryName = abssubdir) t = archive.info.get('general', 'archive-type') if t in archiveTypes: archiveTypes[t][0].append(archive) archiveTypes[t][1].insert(END, archive.name) def addArchive(self, archive): archiveType = archive.info.get('general', 'archive-type') if archiveType=='problem': self.problemsArchives.append(archive) self.problemsListbox.insert(END, archive.name) self.notebook.selectpage('problems') self.problemsListbox.focus() elif archiveType=='solution': self.solutionsArchives.append(archive) self.notebook.selectpage('solutions') self.solutionsListbox.focus() self.solutionsListbox.insert(END, archive.name) self.solutionsListbox.setvalue(archive.name) def __addArchive(self): name = tkSimpleDialog.askstring("Insert name", "Type new archive name (without .tar.gz)") if name: archive = Archive(name) self.addArchive(archive) def __closeApp(self): for archive in self.problemsArchives: archive.close() for archive in self.solutionsArchives: archive.close() LOG.info("Closing application") self.root.destroy() def __loadOptions(self): self.config = DOUGConfigParser() self.config.addConfigContents(_configContent) if os.path.exists('.manager.conf'): self.config.read('.manager.conf') def __saveOptions(self): f = open('.manager.conf', 'w') try: LOG.info('Saving %s' % f) self.config.write(f) finally: f.close() def getProblemArchive(self, name): for archive in self.problemsArchives: if archive.name==name: return archive return None
def __loadOptions(self): self.config = DOUGConfigParser() self.config.addConfigContents(_configContent) if os.path.exists('.manager.conf'): self.config.read('.manager.conf')
def main(testResults): # svn if conf.getboolean('testscript', 'run-svn'): svndoug = svnscripts.run(confFileNames, conf.defaults()) # autotools if conf.getboolean('testscript', 'run-autotools'): autotools.run(confFileNames, conf.defaults()) # construct tests if conf.getboolean('testscript', 'run-tests'): testSuite = unittest.TestSuite() items = conf.items("tests") for name, value in items: if not name.startswith("test"): continue tname = name[4:] # read test data ctrlfname, solutionfname, testconfs = tuple(value.strip().split(" ", 2)) ctrlfname = os.path.abspath(ctrlfname) solutionfname = os.path.abspath(solutionfname) datadir = os.path.dirname(ctrlfname) LOG.debug("Constructing test '%s'" % (tname, )) LOG.debug("Control file: %s" % ctrlfname) LOG.debug("Correct solution file: %s" % (solutionfname,)) # read test configurations testconfs = testconfs.split(",") for testconf in testconfs: testconfname = "testconf_%s" % testconf testname = "%s_%s" % (tname, testconf) solvers = map(int, conf.get(testconfname, "solver").split(",")) methods = map(int, conf.get(testconfname, "method").split(",")) levels = map(int, conf.get(testconfname, "levels").split(",")) fine_methods = map(int, conf.get(testconfname, "fine_methods").split(",")) nsubdomains = map(int, conf.get(testconfname, "num_subdomains").split(",")) processors = map(int, conf.get(testconfname, "processors").split(",")) executables = conf.get(testconfname, "executables").split(",") overlaps = map(int, conf.get(testconfname, "overlaps").split(",")) smoothers = map(int, conf.get(testconfname, "smoothers").split(",")) testtuples = generateTuples(solvers, methods, levels, fine_methods, nsubdomains, processors, executables, overlaps, smoothers) for solver,method,level,fine_method, nsubdomain,nproc,executable,overlap,smoother in testtuples: dougControlFile = doug.execution.ControlFile(filename=ctrlfname, basedir=os.path.dirname(ctrlfname)) dougConfig = DOUGConfigParser(name='DOUG execution parameters') # set/copy doug configuration from tests configuration dougConfig.add_section('doug') for name,value in conf.items('doug', raw=True): dougConfig.set('doug', name, value) dougConfig.add_section('doug-controls') for name,value in conf._sections['doug-controls'].items(): dougConfig.set('doug-controls', name, value) dougConfig.add_section('doug-tests') dougConfig.set('doug-tests', 'csolution_file', solutionfname) # set test configuration dougConfig.set('doug-controls', 'solver', str(solver)) dougConfig.set('doug-controls', 'method', str(method)) dougConfig.set('doug-controls', 'levels', str(level)) dougConfig.set('doug-controls', 'fine_method', str(fine_method)) dougConfig.set('doug-controls', 'num_subdomains', str(nsubdomain)) dougConfig.set('doug', 'nproc', str(nproc)) dougConfig.set('doug', 'executable', executable) dougConfig.set('doug-controls', 'overlap', str(overlap)) dougConfig.set('doug-controls', 'smoothers', str(smoother)) # create DOUG execution object execution = doug.execution.DOUGExecution(dougConfig, dougControlFile) test = dougtest.TestCase(execution, testname) #resultConfig=execution.run() #test = dougtest.MPITestCase(name+"_"+testconf, datadir, ctrlfname, solutionfname, conf, *testtuple) testSuite.addTest(test) # run tests testRunner = dougtest.TestRunner(testResults) testRunner.run(testSuite)