def find_unmatched(self, test): """Does this manager's filters match the given test properties? Returns '' if it does, the filter that failed if not. """ fe = self.filterenv(test) fe['SELF'] = test for f in self.filters: #print 'SAD DEBUG Filter is %s.'% repr(f) #print fe.copy() #print 'SAD END' try: if eval(f, {}, fe.copy()): #print 'SAD DEBUG Filter %s DID pass.'% repr(f) pass else: #print 'SAD DEBUG Filter %s did not pass.'% repr(f) if debug(): log('Filter %s did not pass.' % repr(f)) return f except KeyboardInterrupt: raise except Exception as e: if debug(): log('In filter %s:' % repr(f), e) return f return ''
def run(self, interactiveTests): """Examine the interactive tests and hand them off to the machine. At this point they have been vetted as tests that the machine canRun. """ machine = self.machine unfinished = machine.scheduler.load(interactiveTests) if configuration.options.skip: self.machine.scheduler.reportObstacles(echo=True) log("In skip mode....!") else: log("Beginning test executions") timeStatusReport = time.time() if configuration.options.continueFreq is not None: timeContinuation = time.time() # Convert minutes to seconds continuationStep = int(configuration.options.continueFreq * 60) while unfinished: timeNow = time.time() timePassed = timeNow - timeStatusReport if timePassed >= configuration.options.reportFreq * 60: #os.system("stty sane") terminal("ATS REPORT AT ELAPSED TIME", wallTime()) # log("ATS REPORT AT ELAPSED TIME", wallTime(), echo=True) timeStatusReport = timeNow self.summary(terminal) machine.scheduler.periodicReport() unfinished = machine.scheduler.step() if configuration.options.continueFreq is not None: timeNow = time.time() if (timeNow - timeContinuation) >= continuationStep: self.continuationFile(interactiveTests, True) timeContinuation = timeNow
def testEnded(self, test, status): """Do book-keeping when a job has exited; call noteEnd for machine-specific part. """ from ats import configuration if MachineCore.debugClass: print("DEBUG MachineCore.testEnded invoked cwd= %s " % (os.getcwd())) globalPostrunScript_outname = test.globalPostrunScript_outname globalPostrunScript = test.options.get('globalPostrunScript', None) #verbose = test.options.get('verbose', False) verbose = configuration.options.debug if not (globalPostrunScript == "unset"): here = os.getcwd() os.chdir(test.directory) if os.path.exists(globalPostrunScript): self._executePreOrPostRunScript(globalPostrunScript, test, verbose, globalPostrunScript_outname) else: log("ERROR: globalPostrunScript %s not found" % (globalPostrunScript), echo=True) sys.exit(-1) os.chdir(here) self.numberTestsRunning -= 1 if MachineCore.debugClass or MachineCore.canRunNow_debugClass: print( "DEBUG MachineCore.testEnded decreased self.numberTestsRunning by 1 to %d " % self.numberTestsRunning) #if num_nodes' in test.__dict__: # num_nodes = test.__dict__.get('num_nodes') # self.numberNodesExclusivelyUsed -= num_nodes # print "MachineCore.testEnded decreased self.numberNodesExclusivelyUsed by %d to %d " % \ # (num_nodes, self.numberNodesExclusivelyUsed) if test.numNodesToUse > 0: self.numberNodesExclusivelyUsed -= test.numNodesToUse if MachineCore.debugClass or MachineCore.canRunNow_debugClass: print( "DEBUG MachineCore.testEnded decreased self.numberNodesExclusivelyUsed by %d to %d (max is %d)" % (test.numNodesToUse, self.numberNodesExclusivelyUsed, self.numNodes)) test.set(status, test.elapsedTime()) #note test.status is not necessarily status after this! #see test.expectedResult self.noteEnd(test) #to be defined in children # now close the outputs if test.stdOutLocGet() != 'terminal': test.fileHandleClose() self.scheduler.testEnded(test)
def _executePreOrPostRunScript(self, cmd_line, test, verbose=False, file_name=None, exit=True): """ Function to run a command and display output to screen. The test dictionary is passed in as a string """ #print "AMBYR" #for key in test.__dict__: # print "test key ", key, " is ", test.__dict__[key] #print "ONDRE" my_executable = str(test.__dict__["executable"]) my_commandLine = str(test.__dict__["commandLine"]) my_np = str(test.__dict__["np"]) my_outname = str(test.__dict__["outname"]) my_directory = str(test.__dict__["directory"]) if file_name is not None: execute_ofp = open(file_name, 'w') process = subprocess.Popen(cmd_line + \ " " + '"' + my_executable + '"' + \ " " + '"' + my_np + '"' + \ " " + '"' + my_directory + '"' + \ " " + '"' + my_outname + '"' + \ " " + '"' + my_commandLine + '"', \ shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) # Poll process for new output until finished while True: nextline = process.stdout.readline() if (nextline == '' and process.poll() != None): break if (verbose == True): sys.stdout.write(nextline) # sys.stdout.flush() if file_name is not None: execute_ofp.write(nextline) output = process.communicate()[0] exitCode = process.returncode if file_name is not None: execute_ofp.close() if (exitCode == 0): pass else: if exit: log('%s FATAL RETURN CODE %d Command: %s' % ("ATS", exitCode, cmd_line), echo=True) raise SystemExit(1) return exitCode
def saveResults(self): """Save the state to a file using saveResultsName as file name; if not absolute, put it in the log directory. """ filename = self.saveResultsName if not os.path.isabs(filename): filename = os.path.join(log.directory, filename) log("Saving state to ", filename, echo=True) f = open(filename, 'w') self.printResults(file=f) f.close() self.saveResultsAsXml(logdir=log.directory)
def source(self, *paths, **vocabulary): """Input one or more source files, with optional additional vocabulary. If introspection=f given in the vocabulary, or using define, it should be a function taking one argument and returning any introspective portion of it. """ if debug(): log("source:", ' '.join(paths), echo=True) introspector = vocabulary.get( 'introspection', testEnvironment.get('introspection', standardIntrospection)) for path in paths: self._source(path, introspector, vocabulary)
def sortTests(self): """Sort the tests as to batch or interactive or other. Return two lists.""" # tests that were invalid, filtered, skipped etc. have status not CREATED # This screens out things that recently got set SKIPPED, etc. interactiveTests = [t for t in self.testlist if t.status is CREATED] batchTests = [t for t in self.testlist if t.status is BATCHED] # postcondition if (batchTests and (configuration.options.nobatch or \ configuration.options.allInteractive)): for t in batchTests: log(t, "BATCH sorting error.", echo=True) raise ValueError('batch test(s) should not exist') return interactiveTests, batchTests
def fileOutDelete(self): if os.path.exists(self.outname): try: os.unlink(self.outname) except: log('Not able to delete %s' % self.outname, echo=True, logging=True) if not self.combineOutput: if os.path.exists(self.errname): try: os.unlink(self.errname) except: log('Not able to delete %s' % self.errname, echo=True, logging=True) pass
def postprocess(self): "Call onExitRoutines." for r in self.onExitRoutines: log(" --------- Calling %s --------" % r.__name__, echo=True) try: r(self) except Exception as details: log(details, echo=True) except KeyboardInterrupt: log("Keyboard interrupt while in exit phase, terminating.", echo=True) return log("-------------------------------", echo=True)
def _execute(self, cmd_line, verbose=False, file_name=None, exit=True): """ Function to run a command and display output to screen. """ if file_name is not None: execute_ofp = open(file_name, 'w') process = subprocess.Popen(cmd_line, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) # Poll process for new output until finished while True: nextline = process.stdout.readline() if (nextline == '' and process.poll() != None): break if (verbose == True): sys.stdout.write(nextline) # sys.stdout.flush() if file_name is not None: execute_ofp.write(nextline) output = process.communicate()[0] exitCode = process.returncode if file_name is not None: execute_ofp.close() if (exitCode == 0): pass else: if exit: log('%s FATAL RETURN CODE %d Command: %s' % ("ATS", exitCode, cmd_line), echo=True) raise SystemExit(1) return exitCode
def testEnded(self, test): """Manage scheduling and reporting tasks for a test that ended. Log result for every test but only show certain ones on the terminal. Prune group list if a group is finished. """ echo = self.verbose or (test.status not in (PASSED, EXPECTED)) g = test.group n = len(g) if n == 1: msg = "%5s #%4d %s %s" % \ (test.status, test.serialNumber, test.name, test.message) else: msg = "%s #%d %s %s Group %d #%d of %d" % \ (test.status, test.serialNumber, test.name, test.message, g.number, test.groupSerialNumber, n) log(msg, echo=echo) self.schedule(msg, time.asctime()) self.removeBlock(test) if g.isFinished(): g.recordOutput() self.groups.remove(g)
def filterdefs(text=None): """Add the given text into the environment used for filtering. With no arguments, clear defined list. """ global _filterwith if text is None: log('filterdefs: erasing definitions') _filterwith = [] else: try: d = {} for f in _filterwith: exec(f, d) exec(text, d) except SyntaxError as e: raise AtsError(e) except KeyboardInterrupt: raise except Exception as e: pass if debug(): log('filterdefs:') log.indent() log(text) log.dedent() _filterwith.append(text)
def saveResultsAsXml(self, file=sys.stdout, logdir=None): filename = self.saveXmlResultsName if not os.path.isabs(filename): filename = os.path.join(logdir, filename) log("Saving junit xml to ", filename, echo=True) from ats.reportutils import writePassedTestCase, writeFailedCodeTestCase, writeStatusTestCase passed = [test for test in self.testlist if (test.status is PASSED)] failed = [test for test in self.testlist if (test.status is FAILED)] timedout = [ test for test in self.testlist if (test.status is TIMEDOUT) ] invalid = [test for test in self.testlist if (test.status is INVALID)] halted = [test for test in self.testlist if (test.status is HALTED)] lsferror = [ test for test in self.testlist if (test.status is LSFERROR) ] outf = open(filename, 'w') outf.write('<?xml version="1.0" encoding="UTF-8"?> <testsuites>') outf.write(' <testsuite name="nightly">') for test in passed: writePassedTestCase(outf, test) for test in failed: writeFailedCodeTestCase(outf, test, log.directory) for test in timedout: # There should be logs for timedout tests so this may need more specific output writeStatusTestCase(outf, test, "TIMEDOUT") for test in invalid: # No logs for invalid test cases - just write a message writeStatusTestCase(outf, test, "INVALID") for test in halted: # Are there logs for halted test cases? I think we need to write a message instead. writeStatusTestCase(outf, test, "HALTED") for test in lsferror: writeStatusTestCase(outf, test, "LSFERROR") # Finish off the xml file outf.write(' </testsuite>') outf.write("</testsuites>") outf.close()
def getResults(self): """Returns an attribute dictionary containing the state of this manager suitable for postprocessing. After forming a potential result r, calls any resultsHooks functions (r, manager) """ r = AttributeDict( started=self.started, options=self.options, savedTime=datestamp(long_format=True), collectTimeEnded=self.collectTimeEnded, badlist=self.badlist, filters=self.filters, groups={}, onCollectedRoutines=[f.__name__ for f in self.onCollectedRoutines], onPrioritizedRoutines=[ f.__name__ for f in self.onPrioritizedRoutines ], onExitRoutines=[f.__name__ for f in self.onExitRoutines], onResultsRoutines=[f.__name__ for f in self.onResultsRoutines], ) r.testlist = [t.getResults() for t in self.testlist] if not hasattr(self, 'machine'): return r # never initialized, nothing else of interest. r.inputFiles = self.inputFiles r.verbose = self.verbose r.machine = AttributeDict() r.batchmachine = None for key, value in self.machine.getResults().items(): r.machine[key] = value if self.batchmachine: r.batchmachine = AttributeDict() for key, value in self.batchmachine.getResults().items(): r.batchmachine[key] = value for hook in self.onResultsRoutines: log(' Calling onResults function', hook.__name__, echo=True) hook(r, self) return r
def logStart(self, test, result): "Make appropriate log entries about the test that was started" if result: m1 = "Start" elif configuration.options.skip: m1 = "SKIP " else: m1 = "" if self.verbose or debug(): m1 = "Failed attempting to start" n = len(test.group) my_nn = 0 my_nt = 0 my_ngpu = 0 msgHosts = "" if hasattr(test, 'rs_nodesToUse'): if len(test.rs_nodesToUse) > 0: msgHosts = "Hosts = [ " for host in test.rs_nodesToUse: msgHosts += str(host) + " " msgHosts += "]" if hasattr(test, 'num_nodes'): my_nn = test.num_nodes if hasattr(test, 'nt'): my_nt = test.nt if hasattr(test, 'ngpu'): my_ngpu = test.ngpu if n == 1: if (test.srunRelativeNode >= 0): msg = '%s #%4d r=%d, N=%d-%d, np=%s, %s, %s' % \ (m1, test.serialNumber, test.srunRelativeNode, test.numberOfNodesNeeded, test.numNodesToUse, test.np, time.asctime(), test.name) else: msg = '%s #%4d %s, %s nn=%i, np=%i, nt=%i, ngpu=%i %s' % \ (m1, test.serialNumber, test.name, msgHosts, my_nn, test.np, my_nt, my_ngpu, time.asctime()) else: if (test.srunRelativeNode >= 0): msg = '%s #%4d r=%d, N=%d-%d, np=%s, %s, (Group %d #%d) %s' % \ (m1, test.serialNumber, test.srunRelativeNode, test.numberOfNodesNeeded, test.numNodesToUse, test.np, test.groupNumber, time.asctime(), test.groupSerialNumber, test.name) else: msg = '%s #%4d (Group %d #%d) %s, %s nn=%i, np=%i, nt=%i, ngpu=%i %s' % \ (m1, test.serialNumber, test.groupNumber, test.groupSerialNumber, test.name, msgHosts, my_nn, test.np, my_nt, my_ngpu, time.asctime()) if configuration.options.showGroupStartOnly: echo = (not result) or self.verbose or (test.groupSerialNumber == 1) or test.options.get( 'record', False) else: echo = (result) or self.verbose or test.options.get( 'record', False) log(msg, echo=echo) self.schedule(msg) if self.verbose or debug(): log.indent() log("Executing", test.commandLine) log("in directory", test.directory) #log("with timelimit", test.timelimit) log.dedent()
def finalReport(self): "Write the final report." log.reset() if self.testlist: log(""" ========================================================= ATS RESULTS %s""" % datestamp(long_format=True), echo=True) log('-------------------------------------------------', echo=True) self.report() log('-------------------------------------------------', echo=True) if not configuration.options.skip: log(""" ATS SUMMARY %s""" % datestamp(long_format=True), echo=True) self.summary(log) self._summary2(log)
def filter(self, *filters): "Add filters to the list. Clear list if no arguments." if not filters: self.filters = [] log('Filter list empty.', echo=self.verbose) return for f in filters: try: f = str(f) except Exception: raise AtsError("filter must be convertible to string") if not f: continue try: r = eval(f, {}, {}) except SyntaxError: raise AtsError('Mal-formed filter, %s' % repr(f)) except KeyboardInterrupt: raise except Exception: pass self.filters.append(f) log('Added filter:', repr(f))
def _summary2(self, log): "Additional detail for summary." tlist = [t for t in self.testlist if t.options.get('report', True)] invalid = [ test.name for test in self.testlist if (test.status is INVALID) ] batched = [test.name for test in tlist if (test.status is BATCHED)] skipped = [test.name for test in tlist if (test.status is SKIPPED)] filtered = [test.name for test in tlist if (test.status is FILTERED)] bad = self.badlist if invalid: log("INVALID: %d %s" % (len(invalid) + len(bad), ', '.join(bad + invalid)), echo=True) if batched: log("BATCHED: %d" % len(batched), echo=True) if filtered: log("FILTERED: %d" % len(filtered), echo=True) if skipped: log("SKIPPED: %d" % len(skipped), echo=True)
def report(self): "Log a report, showing each test." doAll = debug() or \ configuration.options.skip or \ configuration.options.verbose outputCaptured = False for test in self.testlist: if test.output: outputCaptured = True if outputCaptured and not configuration.options.hideOutput: log("NOTICE:", "Captured output, see log.", echo=True, logging=False) for test in self.testlist: if doAll or test.notes or test.groupSerialNumber ==1 or \ test.group.echoStatus() or test.options.get('record', False): echo = True else: echo = False log("#%d %s %s %s (Group %d #%d)" % \ (test.serialNumber, test.status, test.name, test.message, test.group.number, test.groupSerialNumber), echo=echo) for line in test.notes: log("NOTE:", line, echo=echo) log.indent() if debug() or configuration.options.skip: log([t.serialNumber for t in test.waitUntil], echo=False) log.dedent()
def logDefinitions(self, *words, **options): """Log the current definitions of words; if none given show all. options passed to log (echo, logging) """ logging = options.get('logging', True) echo = options.get('echo', True) log("Test environment symbols:", logging=logging, echo=echo) log.indent() if not words: words = list(testEnvironment.keys()) words.sort() for key in words: try: log(key, ":", testEnvironment[key], logging=logging, echo=echo) except KeyError: log("Not defined:", key, logging=logging, echo=echo) log.dedent()
def recordOutput(self, groupFailure): """Standard recorder for test. Some tests have a very large amount of output, so this does direct output rather than keep it in the test object. groupFailure is used if one member of a group died. """ failures = [FAILED, TIMEDOUT, HALTED] # What we're going to do with the output file: checkit = self.options.get('check', False) keep = self.options.get('keep') magic = self.options.get('magic', '#ATS:') if checkit: self.notes.append( 'Please check the results of this test manually.') log('Please check the results of this test manually.', echo=True) if configuration.options.skip: return if self.testStdout != 'terminal': try: f = open(self.outname, 'r') except IOError as e: self.notes = ['Missing output file.'] log('Missing output file', self.outname, e) return if magic is not None: # Check for output that starts with magic phrase n = 0 M = len(magic) for line in f: n += 1 if line.startswith(magic): self.output.append(line[M:-1]) f.close() failed = self.status in failures lookatit = checkit or failed keepit = (keep > 0) or lookatit or groupFailure hideIt = self.options.get('hideOutput') if keepit and (self.testStdout != 'terminal'): log.indent() if magic is not None: log('%d lines of output in %s' % (n, self.shortoutname), echo=lookatit) else: log('Output in %s' % self.shortoutname, echo=lookatit) log.dedent() else: self.fileOutDelete() if not hideIt and self.output: log("Captured output:", echo=False, logging=True) log.indent() for line in self.output: log(line, echo=False, logging=True) log.dedent()
def __bool__(self): "It is not proper to test the truth of a test." self.set(FAILED, 'if test(...) not allowed.') log(self, echo=True) return 0
def __init__(self, *fixedargs, **options): "Must not throw an exception -- object must always get created." super(AtsTest, self).__init__() AtsTest.serialNumber += 1 AtsTest.waitUntilAccumulator.append(self) # populate attributes self.serialNumber = AtsTest.serialNumber if AtsTest.group is None: AtsTest.groupCounter += 1 self.group = AtsTestGroup(AtsTest.groupCounter) else: self.group = AtsTest.group self.group.append(self) self.groupNumber = self.group.number self.groupSerialNumber = len(self.group) self.waitUntil = AtsTest.waitUntil #never modify this, it may be shared. self.runOrder = 0 # to aid in diagnosis of wait, priority self.depends_on = None self.dependents = [] self.expectedResult = PASSED self.setName("uninitialized") self.set(INVALID, "New test, unitialized") self.srunRelativeNode = -1 self.numNodesToUse = -1 self.priority = -1 self.totalPriority = -1 self.startDateTime = curDateTime() self.endDateTime = curDateTime() self.output = [] #magic output, newlines and magic removed. self.notes = [] #note from the run self.block = '' # these will all get changed below but want them set to something for getResults self.level = 0 self.independent = False self.np = 1 self.priority = 1 self.totalPriority = 1 self.directory = '' self.batch = False self.clas = '' self.combineOutput = False self.outname = '' self.shortoutname = '' self.errname = '' self.outhandle = None self.errhandle = None self.commandList = ['not run'] # this is just used for documentation self.commandLine = 'not run' rootdict = dict(ATSROOT=configuration.ATSROOT) # Combine the options: first the defaults, then the glued, then the tacked, # then the stuck, then the test options. self.options = AttributeDict( script='', clas=[], executable='', directory='', ) try: self.options.update(configuration.options.testDefaults) self.options.update(AtsTest.glued) self.options.update(AtsTest.tacked) self.options.update(AtsTest.stuck) self.options.update(AtsTest.grouped) self.options.update(options) except Exception as e: self.set(INVALID, 'Bad options: ' + e) return self.level = self.options['level'] self.np = self.options['np'] self.priority = self.options.get('priority', max(1, self.np)) self.totalPriority = self.priority self.testStdout = self.options['testStdout'] outOpts = ['file', 'terminal', 'both'] if not self.testStdout in outOpts: msg = 'Invalid setting for option testStdout: ' + self.testStdout raise AtsError(msg) if configuration.options.allInteractive: self.batch = False else: self.batch = self.options['batch'] if configuration.options.combineOutErr: self.combineOutput = True else: self.combineOutput = False # process the arguments # Note: old interface was script, clas='', **options # Now allow for possibility of no script, or clas as unnamed second # positional lc = len(fixedargs) if lc > 2: self.set(INVALID, 'Too many positional arguments to test command.') return elif lc == 2: self.options['script'] = fixedargs[0] self.options['clas'] = fixedargs[1] elif lc == 1: self.options['script'] = fixedargs[0] script = self.options['script'] clas = self.options['clas'] if isinstance(clas, str): clas = configuration.machine.split(clas) self.clas = [c % self.options for c in clas] executable = str(self.options.get('executable')) self.directory = self.options['directory'] if executable == '1': if not script: self.set(INVALID, "executable = 1 requires a first argument.") return script = script.replace('$ATSROOT', configuration.ATSROOT) if len(configuration.ATSROOT) == 0: script = script[1:] # remove leading "/" or "\" script = script % rootdict self.executable = Executable(script) if self.directory == '': self.directory = os.getcwd() path = self.executable.path junk, filename = os.path.split(path) else: if executable: executable = executable.replace('$ATSROOT', configuration.ATSROOT) self.executable = Executable(executable % rootdict) else: self.executable = configuration.defaultExecutable if script: script = abspath(script) % self.options self.clas.insert(0, script) if self.directory == '': self.directory, filename = os.path.split(script) else: if self.directory == '': self.directory = os.getcwd() junk, filename = os.path.split(self.executable.path) name, junk = os.path.splitext(filename) self.setName(self.options.get('name', name)) label = self.options.get('label', '') if label: label = str(label).strip() self.setName(self.name + '(' + label + ')') if debug(): log("Results of parsing test arguments", echo=False) log.indent() log("Name:", self.name, echo=False) log("Options:", echo=False) log.indent() for k in self.options: log(k, ": ", self.options[k], echo=False) log.dedent() log("Executable path:", self.executable.path, echo=False) log("Directory:", self.directory, echo=False) log.dedent() self.independent = self.options.get('independent', False) if not self.independent: # the lower() is due to peculiarities on at least the Mac # where os.chdir() seems to change case partially. self.block = self.directory.lower() if not self.executable.is_valid(): self.set(INVALID, 'Executable "%s" not valid.' % self.executable) return if not os.path.isdir(self.directory): self.set(INVALID, 'Directory not valid: %s' % self.directory) if script and not is_valid_file(script): self.set(INVALID, "Script %s does not exist." % script) return self.fileOutNamesSet() #set the timelimit try: tl = options.get('timelimit', None) if tl is None: self.timelimit = configuration.timelimit else: self.timelimit = Duration(tl) except AtsError as msg: self.set(INVALID, msg) return if self.priority <= 0: self.set(SKIPPED, 'Test has priority <= zero.') return # if the test ends up BATCHED, such jobs are legal. if self.batch and configuration.options.nobatch: self.set(SKIPPED, "Batch not available") elif self.batch: problem = configuration.batchmachine.canRun(self) if not problem: if configuration.options.skip: self.set(SKIPPED, "BACH skipped due to skip flag") else: self.set(BATCHED, "Ready to run in batch.") else: self.set(SKIPPED, problem) else: problem = configuration.machine.canRun(self) if not problem: self.set(CREATED, "Ready to run interactively.") elif configuration.options.allInteractive or \ configuration.options.nobatch or \ self.groupNumber: self.set(SKIPPED, problem) else: self.set(BATCHED, problem) self.notes.append(\ "Changed to batch since unable to run interactively on this machine.")
def _launch(self, test): """Replace if not using subprocess (unusual). The subprocess part of launch. Also the part that might fail. """ if MachineCore.debugClass: print("DEBUG MachineCore._launch invoked cwd= %s " % os.getcwd()) #print self #print test #print test.options #print test.__dict__ #print self.__dict__ from ats import configuration # See if user specified a file to use as stdin to the test problem. stdin_file = test.options.get('stdin', None) globalPrerunScript_outname = test.globalPrerunScript_outname globalPrerunScript = test.options.get('globalPrerunScript', None) #verbose = test.options.get('verbose', False) verbose = configuration.options.debug if not (globalPrerunScript == "unset"): here = os.getcwd() os.chdir(test.directory) if os.path.exists(globalPrerunScript): self._executePreOrPostRunScript(globalPrerunScript, test, verbose, globalPrerunScript_outname) else: log("ERROR: globalPrerunScript %s not found" % (globalPrerunScript), echo=True) sys.exit(-1) os.chdir(here) try: Eadd = test.options.get('env', None) if Eadd is None: E = None else: # This is old Paul DuBois coding, with ugly syntax. # # That is the syntax for the user within a 'test' or #ATS line is: # env={'ANIMAL': 'duck', 'CITY': 'Seattle', 'PLANET': 'Venus'} # # The apparent reason for this ugliness, is that is how the environment # object is stored in Python. So it amakes the coding easier # here, but it shifts the burden to the user to get that syntax correct, # including the brackets, quotes, commans, and colons. # # Will live with this for now, but would really like this to be more human friendly # # env="ANIMAL=duck, CITY=Seattle, PLANET=Venus" # if MachineCore.debugClass: print("DEBUG MachineCore._launch env specified = %s " % Eadd) E = os.environ.copy() E.update(Eadd) testStdout = test.stdOutLocGet() if stdin_file is None: #print "DEBUG MachineCore._launch 010 " testStdin = None else: #print "DEBUG MachineCore._launch 020 " testStdin = open(test.directory + '/' + stdin_file) # 2016-09-01 # Starting jobs too fast confuses slurm and MPI. Short wait between each job submittal # This showsd up with my atsHello test program # Default sleep is 1 on toss, 0 on other systems, may be set by user on command line # 2016-12-02 # Default sleep is now 0 on all systems. if hasattr(test, 'runningWithinSalloc'): if configuration.options.sleepBeforeSrun > 0: if MachineCore.printSleepBeforeSrunNotice: MachineCore.printSleepBeforeSrunNotice = False print( "ATS Info: MachineCore._launch Will sleep %d seconds before each srun " % configuration.options.sleepBeforeSrun) time.sleep(configuration.options.sleepBeforeSrun) else: if configuration.options.sleepBeforeSrun > 0: if MachineCore.printSleepBeforeSrunNotice: MachineCore.printSleepBeforeSrunNotice = False print( "ATS Info: MachineCore._launch Will sleep %d seconds before each srun " % configuration.options.sleepBeforeSrun) time.sleep(configuration.options.sleepBeforeSrun) if testStdout == 'file': # Get the file handles for standard out and standard error outhandle, errhandle = test.fileHandleGet() # Prepend information about the test to its standard output self.log_prepend(test, test.outhandle) if stdin_file is None: # SAD ambyr #print "DEBUG MachineCore._launch 050 " #print "DEBUG MachineCore._launch %s " % test.commandList #print E #test.child = subprocess.Popen(test.commandList, cwd=test.directory, stdout=outhandle, stderr=errhandle, env=E, text=True) test.child = subprocess.Popen(test.commandList, universal_newlines=True, cwd=test.directory, stdout=outhandle, stderr=errhandle, env=E, text=True) #test.child.wait() else: #print "DEBUG MachineCore._launch 110 " #print testStdin #print "DEBUG MachineCore._launch 120 test.directory = %s" % test.directory test.child = subprocess.Popen(test.commandList, cwd=test.directory, stdout=outhandle, stderr=errhandle, env=E, stdin=testStdin, text=True) elif testStdout == 'terminal': if MachineCore.debugClass: print("DEBUG MachineCore._launch Invoking Popen 2 %s " % test.commandList) if stdin_file is None: test.child = subprocess.Popen(test.commandList, cwd=test.directory, env=E, text=True) else: test.child = subprocess.Popen(test.commandList, cwd=test.directory, env=E, stdin=testStdin, text=True) elif testStdout == 'both': # Get the file handles for standard out and standard error outhandle, errhandle = test.fileHandleGet() # Prepend information about the test to its standard output self.log_prepend(test, test.outhandle) if MachineCore.debugClass: print("DEBUG MachineCore._launch Invoking Popen 3 %s " % test.commandList) if stdin_file is None: test.child = subprocess.Popen(test.commandList, cwd=test.directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=E, text=True) else: test.child = subprocess.Popen(test.commandList, cwd=test.directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=E, stdin=testStdin) test.set(RUNNING, test.commandLine) self.running.append(test) self.numberTestsRunning += 1 if MachineCore.debugClass or MachineCore.canRunNow_debugClass: print( "DEBUG MachineCore.testEnded increased self.numberTestsRunning by 1 to %d " % self.numberTestsRunning) if test.numNodesToUse > 0: self.numberNodesExclusivelyUsed += test.numNodesToUse if MachineCore.debugClass or MachineCore.canRunNow_debugClass: print( "DEBUG MachineCore._launch__ increased self.numberNodesExclusivelyUsed by %d to %d (max is %d)" % (test.numNodesToUse, self.numberNodesExclusivelyUsed, self.numNodes)) return True except OSError as e: if test.stdOutLocGet() != 'terminal': test.fileHandleClose() test.set(FAILED, str(e)) return False
def firstBanner(self): "Write the opening banner." log.echo = True log('ATS START', atsStartTimeLong) log('ATS VERSION', version.version) log('ATS HOST NAME:', socket.gethostname()) log('ATS LOG DIRECTORY:', log.directory) log('SYS_TYPE:', configuration.SYS_TYPE) log('MACHINE_TYPE', configuration.MACHINE_TYPE) log('BATCH_TYPE', configuration.BATCH_TYPE) log("Machine description: ", self.machine.label(), echo=True) if self.batchmachine: log("Batch facility name:", self.batchmachine.label(), echo=True) else: log("No batch facility found.", echo=True) if not configuration.options.logUsage: log('NOT logging usage.') if configuration.options.info or debug(): configuration.documentConfiguration() log.echo = self.verbose if configuration.options.oneFailure: log('Will stop after first failure.') if configuration.options.allInteractive: log('Will run all tests (including any batch tests) as interactive.' ) log('Default time limit for each test=', Duration(configuration.timelimit))
def load(self, batchlist): "Simulate the batch system" log("Simulation of batch load:", echo=True) for t in batchlist: log(t, echo=True)
def init(clas = '', adder = None, examiner=None): """Called by manager.init(class, adder, examiner) Initialize configuration and process command-line options; create log, options, inputFiles, timelimit, machine, and batchmatchine. Call backs to machine and to adder/examiner for options. """ global log, options, inputFiles, timelimit, machine, batchmachine,\ defaultExecutable, ATSROOT, cuttime init_debugClass = False if init_debugClass: print("DEBUG init entered clas=%s " % (clas)) # get the machine and possible batch facility machineDirs = MACHINE_DIR # delete, not needed, handled above #if MACHINE_OVERRIDE_DIR: # machineDirs.append(MACHINE_OVERRIDE_DIR) machineList = [] for machineDir in machineDirs: log('machineDir', machineDir) machineList.extend([os.path.join(machineDir,x) for x in os.listdir(machineDir) if x.endswith('.py') and not x.endswith('__init__.py')]) sys.path.insert(0, machineDir) #machineList = [] #for machineDir in machineDirs: # print("DEBUG machineDir=%s " % (machineDir)) # machineList.extend( # [os.path.join(machineDir,x) for x in os.listdir(machineDir) # if x.endswith('.py') and not x.endswith('__init__.py')]) machine = None batchmachine = None specFoundIn = '' bspecFoundIn = '' if init_debugClass: print("DEBUG init 100") print(machineDirs) print(machineList) print(MACHINE_TYPE) print("DEBUG init 200") for full_path in machineList: moduleName = '' fname = os.path.basename(full_path) # print "DEBUG 000 fname = %s" % fname f = open(full_path, 'r') for line in f: if line.startswith('#ATS:') and not machine: items = line[5:-1].split() machineName, moduleName, machineClass, npMaxH = items if init_debugClass: print("DEBUG init machineName=%s moduleName=%s machineClass=%s npMaxH=%s" % (machineName, moduleName, machineClass, npMaxH)) # print "DEBUG init MACHINE_TYPE=%s machineName=%s moduleName=%s machineClass=%s npMaxH=%s" % (MACHINE_TYPE, machineName, moduleName, machineClass, npMaxH) if machineName == MACHINE_TYPE: if moduleName == "SELF": moduleName, junk = os.path.splitext(fname) specFoundIn = full_path print(f"from ats.atsMachines.{moduleName} " f"import {machineClass} as Machine") try: machine_factory = get_machine_factory(moduleName, machineClass) except ModuleNotFoundError: machine_factory = get_machine_factory(moduleName, machineClass, machine_package='atsMachines') machine = machine_factory(machineName, int(npMaxH)) elif line.startswith('#BATS:') and not batchmachine: items = line[6:-1].split() machineName, moduleName, machineClass, npMaxH = items if machineName == BATCH_TYPE: if moduleName == "SELF": moduleName, junk = os.path.splitext(fname) bspecFoundIn = full_path try: machine_factory = get_machine_factory(moduleName, machineClass) except ModuleNotFoundError: machine_factory = get_machine_factory(moduleName, machineClass, machine_package='atsMachines') batchmachine = machine_factory(moduleName, int(npMaxH)) f.close() if machine and batchmachine: break if machine is None: terminal("No machine specifications for", SYS_TYPE, "found, using generic.") machine = machines.Machine('generic', -1) # create the option set usage = "usage: %prog [options] [input files]" parser = OptionParser(usage=usage, version="%prog " + version.version) addOptions(parser) machine.addOptions(parser) # add the --nobatch option but force it true if no batch facility here. parser.add_option('--nobatch', action='store_true', dest='nobatch', default=(batchmachine is None), help = 'Do not run batch jobs.') if batchmachine: batchmachine.addOptions(parser) # user callback? if adder is not None: adder(parser) # parse the command line if clas: import shlex argv = shlex.split(clas) else: argv = sys.argv[1:] (toptions, inputFiles) = parser.parse_args(argv) # immediately make the options a real dictionary -- the way optparse leaves it # is misleading. options = AttributeDict() for k in vars(toptions).keys(): options[k] = getattr(toptions, k) # set up the test default options so the machine(s) can add to it options['testDefaults'] = AttributeDict(np=1, batch=0, level=1, keep = options.keep, hideOutput = options.hideOutput, verbose = options.verbose, testStdout = options.testStdout, globalPrerunScript = options.globalPrerunScript, globalPostrunScript = options.globalPostrunScript, sequential = options.sequential, nosrun = options.nosrun, salloc = options.salloc ) # let the machine(s) modify the results or act upon them in other ways. machine.examineOptions(options) if batchmachine: batchmachine.examineOptions(options) # unpack basic options debug(options.debug) if options.logdir: log.set(directory = options.logdir) else: dirname = SYS_TYPE + "." + atsStartTime + ".logs" log.set(directory = dirname) log.mode="w" log.logging = 1 # user callback? if examiner is not None: examiner(options) if specFoundIn: log("Found specification for", MACHINE_TYPE, "in", specFoundIn) else: log("No specification found for", MACHINE_TYPE, ', using generic') if bspecFoundIn: log("Batch specification for ", BATCH_TYPE, "in", bspecFoundIn) # unpack other options cuttime = options.cuttime if cuttime is not None: cuttime = Duration(cuttime) timelimit = Duration(options.timelimit) defaultExecutable = executables.Executable(abspath(options.executable)) # ATSROOT is used in tests.py to allow paths pointed at the executable's directory if 'ATSROOT' in os.environ: ATSROOT = os.environ['ATSROOT'] else: ATSROOT = os.path.dirname(defaultExecutable.path)
def documentConfiguration(): """Write the configuration to the log.""" log('Configuration:') log.indent() log('Input files:', inputFiles) log('Python path:', sys.executable) log('ATS from ', os.path.dirname(__file__)) log('ATS version:', version.version) log('Options:') log.indent() olist = list(options.keys()) olist.sort() for k in olist: log(k + ":", repr(getattr(options, k))) log.dedent() log.dedent()
def core(self): "This is the 'guts' of ATS." if configuration.SYS_TYPE == "toss_3_x86_64": if configuration.options.bypassSerialMachineCheck == False: log("**********************************************************************************", echo=True) log("*** This is a serial machine --- Do not use ATS on more than 1 node here! ***", echo=True) log("*** ***", echo=True) log("*** Use ATS option --bypassSerialMachineCheck if you promise to run on 1 Node ***", echo=True) log("**********************************************************************************", echo=True) sys.exit(-1) # Phase 1 -- collect the tests errorOccurred = False try: # surround with keyboard interrupt, AtsError handlers self.collectTests() except AtsError: log("ATS error while collecting tests.", echo=True) log(traceback.format_exc(), echo=True) errorOccurred = True self.collectTimeEnded = datestamp(long_format=True) except KeyboardInterrupt: log("Keyboard interrupt while collecting tests, terminating.", echo=True) errorOccurred = True self.collectTimeEnded = datestamp(long_format=True) if errorOccurred: return try: for f in self.onCollectedRoutines: log("Calling onCollected routine", f.__name__, echo=self.verbose) f(self) except KeyboardInterrupt: log("Keyboard interrupt while collecting tests, terminating.", echo=True) errorOccurred = True except Exception: log("Error in user-specified onCollected routine.", echo=True) log(traceback.format_exc(), echo=True) errorOccured = True if errorOccurred: return # divide into interactive and batch tests interactiveTests, batchTests = self.sortTests() if len(interactiveTests) + len(batchTests) == 0: log("No tests found.", echo=True) return # We have built up the list of tests. Run functions added via # beforeRun() calls to allow user to do stuff like cleaning up old test # results before running or other things. self.preprocess() # Phase 2 -- dispatch the batch tests if self.batchmachine and batchTests: if configuration.options.skip: log("Skipping execution due to --skip") else: try: log("Sending %d tests to %s." % (len(batchTests), self.batchmachine.name), echo=True) self.batchmachine.load(batchTests) except AtsError: log(traceback.format_exc(), echo=True) log("ATS error.", echo=True) return except KeyboardInterrupt: log("Keyboard interrupt while dispatching batch, terminating.", echo=True) return # Phase 3 -- run the interactive tests dieDieDie = False if interactiveTests: self.machine.scheduler.prioritize(interactiveTests) try: log("Total number of interactive tests = ", len(interactiveTests), echo=True) log("---------------------------------------------------", echo=True) for f in self.onPrioritizedRoutines: log("Calling onPrioritized routine", f.__name__, echo=self.verbose) f(interactiveTests) except KeyboardInterrupt: log("Keyboard interrupt while prioritizing tests, terminating.", echo=True) errorOccurred = True except Exception: log("Error in prioritizing tests.", echo=True) log(traceback.format_exc(), echo=True) errorOccured = True if errorOccurred: return try: self.run(interactiveTests) except AtsError: log(traceback.format_exc(), echo=True) log("ATS error. Removing running jobs....", echo=True) dieDieDie = True except KeyboardInterrupt: dieDieDie = True log("Keyboard interrupt. Removing running jobs....", echo=True) if dieDieDie: time.sleep(3) for test in self.testlist: if (test.status is RUNNING): self.machine.kill(test) self.machine.quit() #machine shutdown / cleanup # Phase 4 -- Continuation file # for t in interactiveTests: # if t.status not in (PASSED, EXPECTED, FILTERED): # break # else: # self.continuationFileName = '' # return self.continuationFile(interactiveTests)
def launch(self, test): """Start executable using a suitable command. Return True if able to do so. Call noteLaunch if launch succeeded.""" from ats import configuration #print test.__dict__ ##print self.__dict__ nosrun = test.options.get('nosrun', False) serial = test.options.get( 'serial', False ) # support serial=True on a per-test basis for backwards compatability for a while if nosrun == True or serial == True: test.commandList = self.calculateBasicCommandList(test) test.cpus_per_task = 1 else: test.commandList = self.calculateCommandList(test) if test.commandList == None: log("ATS def launch returning false, commandList is None", echo=True) return False # # On Blueos (Sierra/Ansel) Set JSM_JSRUN_NO_WARN_OVERSUBSCRIBE the same as lrun does # if configuration.SYS_TYPE.startswith('blueos'): os.environ['JSM_JSRUN_NO_WARN_OVERSUBSCRIBE'] = '1' # To enable running of threaded codes in 1 thread mode, the OMP_NUM_THREADS must be # set either by the user before the run, or by the test 'nt' option, or by # the command line option to ATS --ompNumThreads. If none of these are set, then # set it to 1. if configuration.options.ompNumThreads > 0: # Priority 1 setting, ats command line if configuration.options.verbose: print( "ATS launch setting OMP_NUM_THREADS %d as user specified --ompNumThreads=%d" % (configuration.options.ompNumThreads, configuration.options.ompNumThreads)) os.environ['OMP_NUM_THREADS'] = str( configuration.options.ompNumThreads) else: # Priority 2 setting, within an ATS test line omp_num_threads = test.options.get('nt', -1) if (omp_num_threads > 0): if configuration.options.verbose: print( "ATS launch setting OMP_NUM_THREADS %d based on test 'nt'option" % omp_num_threads) os.environ['OMP_NUM_THREADS'] = str(omp_num_threads) else: # Priority 3 setting, the user has already set OMP_NUM_THREADS in their environment if 'OMP_NUM_THREADS' in os.environ: if configuration.options.verbose: temp_omp = os.getenv("OMP_NUM_THREADS") # print "ATS detected that OMP_NUM_THREADS is already set to %s" % (temp_omp) # Priority 4 setting, set it to 1 if it is not othewise set else: if configuration.options.verbose: print( "ATS launch setting OMP_NUM_THREADS 1 by default for as it was not specified for the test." ) # print " This should allow for threaded applications to run with non threaded tests with a single thread." os.environ['OMP_NUM_THREADS'] = str(1) # Set default KMP_AFFINITY so that OpenMP runs are OK on Toss 3 # This is experimental for now. if configuration.SYS_TYPE.startswith('toss'): if MachineCore.printExperimentalNotice: MachineCore.printExperimentalNotice = False print("ATS Experimental: setting KMP_AFFINITY to %s on Toss" % configuration.options.kmpAffinity) os.environ['KMP_AFFINITY'] = configuration.options.kmpAffinity # Turn off shared memory mpi collective operations on toss and chaos if configuration.SYS_TYPE.startswith('toss'): os.environ['VIADEV_USE_SHMEM_COLL'] = "0" if configuration.SYS_TYPE.startswith('chaos'): os.environ['VIADEV_USE_SHMEM_COLL'] = "0" # LS_COLORS can mess up somesystem and is not needed for any platform by ATS os.environ['LS_COLORS'] = "" # Bamboo env vars can also mess up somesystem runs by exceeding char limit for env vars # remove them os.environ['bamboo_shortJobName'] = "" os.environ['bamboo_capability_system_git_executable'] = "" os.environ['bamboo_build_working_directory'] = "" os.environ['bamboo_shortPlanKey'] = "" os.environ['bamboo_planName'] = "" os.environ['bamboo_capability_system_jdk_JDK_1_8_0_71'] = "" os.environ['bamboo_buildKey'] = "" os.environ['bamboo_capability_system_jdk_JDK'] = "" os.environ['bamboo_capability_sys_type'] = "" os.environ['bamboo_capability_cluster'] = "" os.environ['bamboo_buildFailed'] = "" os.environ['bamboo_buildResultKey'] = "" os.environ['bamboo_plan_storageTag'] = "" os.environ['bamboo_planKey'] = "" os.environ['bamboo_capability_system_builder_ant_Ant'] = "" os.environ['bamboo_shortPlanName'] = "" os.environ['bamboo_buildResultsUrl'] = "" os.environ['bamboo_buildPlanName'] = "" os.environ['bamboo_working_directory'] = "" os.environ['bamboo_agentWorkingDirectory'] = "" os.environ['bamboo_buildTimeStamp'] = "" os.environ['bamboo_shortJobKey'] = "" os.environ['bamboo_buildNumber'] = "" os.environ['bamboo_agentId'] = "" os.environ['bamboo_capability_system_jdk_JDK_1_8'] = "" os.environ['bamboo_resultsUrl'] = "" test.commandLine = " ".join(test.commandList) sandbox = test.options.get('sandbox', False) directory = test.options.get('directory', None) deck_directory = test.options.get('deck_directory', None) if sandbox: # directory is the name of the sandbox directory if directory == None or directory == '': directory = ('%s_%d_%04d_%s') % ( 'sandbox', os.getpid(), test.serialNumber, test.namebase) if deck_directory == None or deck_directory == '': deck_directory = os.getcwd() if not os.path.isdir(directory): if MachineCore.debugClass: print( "MachineCore.launch \n\tcwd=%s \n\tdir=%s \n\tdeck_directory=%s" % (os.getcwd(), directory, deck_directory)) log("ATS machines.py Creating sandbox directory : %s" % directory, echo=True) copytree(deck_directory, directory, ignore=ignore_patterns('*.logs', 'html', '.svn', '*sandbox*')) #--- placing this here doesn't allow the machines to handle the skip option themselves.. if configuration.options.skip: test.set(SKIPPED, "--skip option") return False test.setStartDateTime() result = self._launch(test) if result: self.noteLaunch(test) return result