def find_unmatched(self, test):
"""Does this manager's filters match the given test properties?
Returns '' if it does, the filter that failed if not.
"""
fe = self.filterenv(test)
fe['SELF'] = test
for f in self.filters:
#print 'SAD DEBUG Filter is %s.'% repr(f)
#print fe.copy()
#print 'SAD END'
try:
if eval(f, {}, fe.copy()):
#print 'SAD DEBUG Filter %s DID pass.'% repr(f)
pass
else:
#print 'SAD DEBUG Filter %s did not pass.'% repr(f)
if debug():
log('Filter %s did not pass.' % repr(f))
return f
except KeyboardInterrupt:
raise
except Exception as e:
if debug():
log('In filter %s:' % repr(f), e)
return f
return ''
def run(self, interactiveTests):
"""Examine the interactive tests and hand them off to the machine.
At this point they have been vetted as tests that the machine canRun.
"""
machine = self.machine
unfinished = machine.scheduler.load(interactiveTests)
if configuration.options.skip:
self.machine.scheduler.reportObstacles(echo=True)
log("In skip mode....!")
else:
log("Beginning test executions")
timeStatusReport = time.time()
if configuration.options.continueFreq is not None:
timeContinuation = time.time()
# Convert minutes to seconds
continuationStep = int(configuration.options.continueFreq * 60)
while unfinished:
timeNow = time.time()
timePassed = timeNow - timeStatusReport
if timePassed >= configuration.options.reportFreq * 60:
#os.system("stty sane")
terminal("ATS REPORT AT ELAPSED TIME", wallTime())
# log("ATS REPORT AT ELAPSED TIME", wallTime(), echo=True)
timeStatusReport = timeNow
self.summary(terminal)
machine.scheduler.periodicReport()
unfinished = machine.scheduler.step()
if configuration.options.continueFreq is not None:
timeNow = time.time()
if (timeNow - timeContinuation) >= continuationStep:
self.continuationFile(interactiveTests, True)
timeContinuation = timeNow
def testEnded(self, test, status):
"""Do book-keeping when a job has exited;
call noteEnd for machine-specific part.
"""
from ats import configuration
if MachineCore.debugClass:
print("DEBUG MachineCore.testEnded invoked cwd= %s " %
(os.getcwd()))
globalPostrunScript_outname = test.globalPostrunScript_outname
globalPostrunScript = test.options.get('globalPostrunScript', None)
#verbose = test.options.get('verbose', False)
verbose = configuration.options.debug
if not (globalPostrunScript == "unset"):
here = os.getcwd()
os.chdir(test.directory)
if os.path.exists(globalPostrunScript):
self._executePreOrPostRunScript(globalPostrunScript, test,
verbose,
globalPostrunScript_outname)
else:
log("ERROR: globalPostrunScript %s not found" %
(globalPostrunScript),
echo=True)
sys.exit(-1)
os.chdir(here)
self.numberTestsRunning -= 1
if MachineCore.debugClass or MachineCore.canRunNow_debugClass:
print(
"DEBUG MachineCore.testEnded decreased self.numberTestsRunning by 1 to %d "
% self.numberTestsRunning)
#if num_nodes' in test.__dict__:
# num_nodes = test.__dict__.get('num_nodes')
# self.numberNodesExclusivelyUsed -= num_nodes
# print "MachineCore.testEnded decreased self.numberNodesExclusivelyUsed by %d to %d " % \
# (num_nodes, self.numberNodesExclusivelyUsed)
if test.numNodesToUse > 0:
self.numberNodesExclusivelyUsed -= test.numNodesToUse
if MachineCore.debugClass or MachineCore.canRunNow_debugClass:
print(
"DEBUG MachineCore.testEnded decreased self.numberNodesExclusivelyUsed by %d to %d (max is %d)"
% (test.numNodesToUse, self.numberNodesExclusivelyUsed,
self.numNodes))
test.set(status, test.elapsedTime())
#note test.status is not necessarily status after this!
#see test.expectedResult
self.noteEnd(test) #to be defined in children
# now close the outputs
if test.stdOutLocGet() != 'terminal':
test.fileHandleClose()
self.scheduler.testEnded(test)
def _executePreOrPostRunScript(self,
cmd_line,
test,
verbose=False,
file_name=None,
exit=True):
"""
Function to run a command and display output to screen. The test dictionary is passed in as a string
"""
#print "AMBYR"
#for key in test.__dict__:
# print "test key ", key, " is ", test.__dict__[key]
#print "ONDRE"
my_executable = str(test.__dict__["executable"])
my_commandLine = str(test.__dict__["commandLine"])
my_np = str(test.__dict__["np"])
my_outname = str(test.__dict__["outname"])
my_directory = str(test.__dict__["directory"])
if file_name is not None:
execute_ofp = open(file_name, 'w')
process = subprocess.Popen(cmd_line + \
" " + '"' + my_executable + '"' + \
" " + '"' + my_np + '"' + \
" " + '"' + my_directory + '"' + \
" " + '"' + my_outname + '"' + \
" " + '"' + my_commandLine + '"', \
shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
# Poll process for new output until finished
while True:
nextline = process.stdout.readline()
if (nextline == '' and process.poll() != None):
break
if (verbose == True):
sys.stdout.write(nextline)
# sys.stdout.flush()
if file_name is not None:
execute_ofp.write(nextline)
output = process.communicate()[0]
exitCode = process.returncode
if file_name is not None:
execute_ofp.close()
if (exitCode == 0):
pass
else:
if exit:
log('%s FATAL RETURN CODE %d Command: %s' %
("ATS", exitCode, cmd_line),
echo=True)
raise SystemExit(1)
return exitCode
def saveResults(self):
"""Save the state to a file using saveResultsName as file name;
if not absolute, put it in the log directory.
"""
filename = self.saveResultsName
if not os.path.isabs(filename):
filename = os.path.join(log.directory, filename)
log("Saving state to ", filename, echo=True)
f = open(filename, 'w')
self.printResults(file=f)
f.close()
self.saveResultsAsXml(logdir=log.directory)
def source(self, *paths, **vocabulary):
"""Input one or more source files, with optional additional vocabulary.
If introspection=f given in the vocabulary, or using define,
it should be a function taking one argument and returning any
introspective portion of it.
"""
if debug():
log("source:", ' '.join(paths), echo=True)
introspector = vocabulary.get(
'introspection',
testEnvironment.get('introspection', standardIntrospection))
for path in paths:
self._source(path, introspector, vocabulary)
def sortTests(self):
"""Sort the tests as to batch or interactive or other. Return two lists."""
# tests that were invalid, filtered, skipped etc. have status not CREATED
# This screens out things that recently got set SKIPPED, etc.
interactiveTests = [t for t in self.testlist if t.status is CREATED]
batchTests = [t for t in self.testlist if t.status is BATCHED]
# postcondition
if (batchTests and (configuration.options.nobatch or \
configuration.options.allInteractive)):
for t in batchTests:
log(t, "BATCH sorting error.", echo=True)
raise ValueError('batch test(s) should not exist')
return interactiveTests, batchTests
def fileOutDelete(self):
if os.path.exists(self.outname):
try:
os.unlink(self.outname)
except:
log('Not able to delete %s' % self.outname,
echo=True,
logging=True)
if not self.combineOutput:
if os.path.exists(self.errname):
try:
os.unlink(self.errname)
except:
log('Not able to delete %s' % self.errname,
echo=True,
logging=True)
pass
def postprocess(self):
"Call onExitRoutines."
for r in self.onExitRoutines:
log(" --------- Calling %s --------" % r.__name__, echo=True)
try:
r(self)
except Exception as details:
log(details, echo=True)
except KeyboardInterrupt:
log("Keyboard interrupt while in exit phase, terminating.",
echo=True)
return
log("-------------------------------", echo=True)
def _execute(self, cmd_line, verbose=False, file_name=None, exit=True):
"""
Function to run a command and display output to screen.
"""
if file_name is not None:
execute_ofp = open(file_name, 'w')
process = subprocess.Popen(cmd_line,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True)
# Poll process for new output until finished
while True:
nextline = process.stdout.readline()
if (nextline == '' and process.poll() != None):
break
if (verbose == True):
sys.stdout.write(nextline)
# sys.stdout.flush()
if file_name is not None:
execute_ofp.write(nextline)
output = process.communicate()[0]
exitCode = process.returncode
if file_name is not None:
execute_ofp.close()
if (exitCode == 0):
pass
else:
if exit:
log('%s FATAL RETURN CODE %d Command: %s' %
("ATS", exitCode, cmd_line),
echo=True)
raise SystemExit(1)
return exitCode
def testEnded(self, test):
"""Manage scheduling and reporting tasks for a test that ended.
Log result for every test but only show certain ones on the terminal.
Prune group list if a group is finished.
"""
echo = self.verbose or (test.status not in (PASSED, EXPECTED))
g = test.group
n = len(g)
if n == 1:
msg = "%5s #%4d %s %s" % \
(test.status, test.serialNumber, test.name, test.message)
else:
msg = "%s #%d %s %s Group %d #%d of %d" % \
(test.status, test.serialNumber, test.name, test.message,
g.number, test.groupSerialNumber, n)
log(msg, echo=echo)
self.schedule(msg, time.asctime())
self.removeBlock(test)
if g.isFinished():
g.recordOutput()
self.groups.remove(g)
def filterdefs(text=None):
"""Add the given text into the environment
used for filtering.
With no arguments, clear defined list.
"""
global _filterwith
if text is None:
log('filterdefs: erasing definitions')
_filterwith = []
else:
try:
d = {}
for f in _filterwith:
exec(f, d)
exec(text, d)
except SyntaxError as e:
raise AtsError(e)
except KeyboardInterrupt:
raise
except Exception as e:
pass
if debug():
log('filterdefs:')
log.indent()
log(text)
log.dedent()
_filterwith.append(text)
def saveResultsAsXml(self, file=sys.stdout, logdir=None):
filename = self.saveXmlResultsName
if not os.path.isabs(filename):
filename = os.path.join(logdir, filename)
log("Saving junit xml to ", filename, echo=True)
from ats.reportutils import writePassedTestCase, writeFailedCodeTestCase, writeStatusTestCase
passed = [test for test in self.testlist if (test.status is PASSED)]
failed = [test for test in self.testlist if (test.status is FAILED)]
timedout = [
test for test in self.testlist if (test.status is TIMEDOUT)
]
invalid = [test for test in self.testlist if (test.status is INVALID)]
halted = [test for test in self.testlist if (test.status is HALTED)]
lsferror = [
test for test in self.testlist if (test.status is LSFERROR)
]
outf = open(filename, 'w')
outf.write('<?xml version="1.0" encoding="UTF-8"?> <testsuites>')
outf.write(' <testsuite name="nightly">')
for test in passed:
writePassedTestCase(outf, test)
for test in failed:
writeFailedCodeTestCase(outf, test, log.directory)
for test in timedout:
# There should be logs for timedout tests so this may need more specific output
writeStatusTestCase(outf, test, "TIMEDOUT")
for test in invalid:
# No logs for invalid test cases - just write a message
writeStatusTestCase(outf, test, "INVALID")
for test in halted:
# Are there logs for halted test cases? I think we need to write a message instead.
writeStatusTestCase(outf, test, "HALTED")
for test in lsferror:
writeStatusTestCase(outf, test, "LSFERROR")
# Finish off the xml file
outf.write(' </testsuite>')
outf.write("</testsuites>")
outf.close()
def getResults(self):
"""Returns an attribute dictionary containing the state of this
manager suitable for postprocessing. After forming a potential
result r, calls any resultsHooks functions (r, manager)
"""
r = AttributeDict(
started=self.started,
options=self.options,
savedTime=datestamp(long_format=True),
collectTimeEnded=self.collectTimeEnded,
badlist=self.badlist,
filters=self.filters,
groups={},
onCollectedRoutines=[f.__name__ for f in self.onCollectedRoutines],
onPrioritizedRoutines=[
f.__name__ for f in self.onPrioritizedRoutines
],
onExitRoutines=[f.__name__ for f in self.onExitRoutines],
onResultsRoutines=[f.__name__ for f in self.onResultsRoutines],
)
r.testlist = [t.getResults() for t in self.testlist]
if not hasattr(self, 'machine'):
return r # never initialized, nothing else of interest.
r.inputFiles = self.inputFiles
r.verbose = self.verbose
r.machine = AttributeDict()
r.batchmachine = None
for key, value in self.machine.getResults().items():
r.machine[key] = value
if self.batchmachine:
r.batchmachine = AttributeDict()
for key, value in self.batchmachine.getResults().items():
r.batchmachine[key] = value
for hook in self.onResultsRoutines:
log(' Calling onResults function', hook.__name__, echo=True)
hook(r, self)
return r
def logStart(self, test, result):
"Make appropriate log entries about the test that was started"
if result:
m1 = "Start"
elif configuration.options.skip:
m1 = "SKIP "
else:
m1 = ""
if self.verbose or debug():
m1 = "Failed attempting to start"
n = len(test.group)
my_nn = 0
my_nt = 0
my_ngpu = 0
msgHosts = ""
if hasattr(test, 'rs_nodesToUse'):
if len(test.rs_nodesToUse) > 0:
msgHosts = "Hosts = [ "
for host in test.rs_nodesToUse:
msgHosts += str(host) + " "
msgHosts += "]"
if hasattr(test, 'num_nodes'):
my_nn = test.num_nodes
if hasattr(test, 'nt'):
my_nt = test.nt
if hasattr(test, 'ngpu'):
my_ngpu = test.ngpu
if n == 1:
if (test.srunRelativeNode >= 0):
msg = '%s #%4d r=%d, N=%d-%d, np=%s, %s, %s' % \
(m1, test.serialNumber, test.srunRelativeNode, test.numberOfNodesNeeded, test.numNodesToUse, test.np, time.asctime(), test.name)
else:
msg = '%s #%4d %s, %s nn=%i, np=%i, nt=%i, ngpu=%i %s' % \
(m1, test.serialNumber, test.name, msgHosts, my_nn, test.np, my_nt, my_ngpu, time.asctime())
else:
if (test.srunRelativeNode >= 0):
msg = '%s #%4d r=%d, N=%d-%d, np=%s, %s, (Group %d #%d) %s' % \
(m1, test.serialNumber, test.srunRelativeNode, test.numberOfNodesNeeded, test.numNodesToUse, test.np, test.groupNumber, time.asctime(), test.groupSerialNumber, test.name)
else:
msg = '%s #%4d (Group %d #%d) %s, %s nn=%i, np=%i, nt=%i, ngpu=%i %s' % \
(m1, test.serialNumber, test.groupNumber, test.groupSerialNumber, test.name, msgHosts, my_nn, test.np, my_nt, my_ngpu, time.asctime())
if configuration.options.showGroupStartOnly:
echo = (not result) or self.verbose or (test.groupSerialNumber
== 1) or test.options.get(
'record', False)
else:
echo = (result) or self.verbose or test.options.get(
'record', False)
log(msg, echo=echo)
self.schedule(msg)
if self.verbose or debug():
log.indent()
log("Executing", test.commandLine)
log("in directory", test.directory)
#log("with timelimit", test.timelimit)
log.dedent()
def finalReport(self):
"Write the final report."
log.reset()
if self.testlist:
log("""
=========================================================
ATS RESULTS %s""" % datestamp(long_format=True),
echo=True)
log('-------------------------------------------------', echo=True)
self.report()
log('-------------------------------------------------', echo=True)
if not configuration.options.skip:
log("""
ATS SUMMARY %s""" % datestamp(long_format=True), echo=True)
self.summary(log)
self._summary2(log)
def filter(self, *filters):
"Add filters to the list. Clear list if no arguments."
if not filters:
self.filters = []
log('Filter list empty.', echo=self.verbose)
return
for f in filters:
try:
f = str(f)
except Exception:
raise AtsError("filter must be convertible to string")
if not f:
continue
try:
r = eval(f, {}, {})
except SyntaxError:
raise AtsError('Mal-formed filter, %s' % repr(f))
except KeyboardInterrupt:
raise
except Exception:
pass
self.filters.append(f)
log('Added filter:', repr(f))
def _summary2(self, log):
"Additional detail for summary."
tlist = [t for t in self.testlist if t.options.get('report', True)]
invalid = [
test.name for test in self.testlist if (test.status is INVALID)
]
batched = [test.name for test in tlist if (test.status is BATCHED)]
skipped = [test.name for test in tlist if (test.status is SKIPPED)]
filtered = [test.name for test in tlist if (test.status is FILTERED)]
bad = self.badlist
if invalid:
log("INVALID: %d %s" %
(len(invalid) + len(bad), ', '.join(bad + invalid)),
echo=True)
if batched:
log("BATCHED: %d" % len(batched), echo=True)
if filtered:
log("FILTERED: %d" % len(filtered), echo=True)
if skipped:
log("SKIPPED: %d" % len(skipped), echo=True)
def report(self):
"Log a report, showing each test."
doAll = debug() or \
configuration.options.skip or \
configuration.options.verbose
outputCaptured = False
for test in self.testlist:
if test.output:
outputCaptured = True
if outputCaptured and not configuration.options.hideOutput:
log("NOTICE:",
"Captured output, see log.",
echo=True,
logging=False)
for test in self.testlist:
if doAll or test.notes or test.groupSerialNumber ==1 or \
test.group.echoStatus() or test.options.get('record', False):
echo = True
else:
echo = False
log("#%d %s %s %s (Group %d #%d)" % \
(test.serialNumber, test.status, test.name, test.message,
test.group.number, test.groupSerialNumber),
echo=echo)
for line in test.notes:
log("NOTE:", line, echo=echo)
log.indent()
if debug() or configuration.options.skip:
log([t.serialNumber for t in test.waitUntil], echo=False)
log.dedent()
def logDefinitions(self, *words, **options):
"""Log the current definitions of words; if none given show all.
options passed to log (echo, logging)
"""
logging = options.get('logging', True)
echo = options.get('echo', True)
log("Test environment symbols:", logging=logging, echo=echo)
log.indent()
if not words:
words = list(testEnvironment.keys())
words.sort()
for key in words:
try:
log(key, ":", testEnvironment[key], logging=logging, echo=echo)
except KeyError:
log("Not defined:", key, logging=logging, echo=echo)
log.dedent()
def recordOutput(self, groupFailure):
"""Standard recorder for test. Some tests have a very large amount of output,
so this does direct output rather than keep it in the test object.
groupFailure is used if one member of a group died.
"""
failures = [FAILED, TIMEDOUT, HALTED]
# What we're going to do with the output file:
checkit = self.options.get('check', False)
keep = self.options.get('keep')
magic = self.options.get('magic', '#ATS:')
if checkit:
self.notes.append(
'Please check the results of this test manually.')
log('Please check the results of this test manually.', echo=True)
if configuration.options.skip:
return
if self.testStdout != 'terminal':
try:
f = open(self.outname, 'r')
except IOError as e:
self.notes = ['Missing output file.']
log('Missing output file', self.outname, e)
return
if magic is not None: # Check for output that starts with magic phrase
n = 0
M = len(magic)
for line in f:
n += 1
if line.startswith(magic):
self.output.append(line[M:-1])
f.close()
failed = self.status in failures
lookatit = checkit or failed
keepit = (keep > 0) or lookatit or groupFailure
hideIt = self.options.get('hideOutput')
if keepit and (self.testStdout != 'terminal'):
log.indent()
if magic is not None:
log('%d lines of output in %s' % (n, self.shortoutname),
echo=lookatit)
else:
log('Output in %s' % self.shortoutname, echo=lookatit)
log.dedent()
else:
self.fileOutDelete()
if not hideIt and self.output:
log("Captured output:", echo=False, logging=True)
log.indent()
for line in self.output:
log(line, echo=False, logging=True)
log.dedent()
def __bool__(self):
"It is not proper to test the truth of a test."
self.set(FAILED, 'if test(...) not allowed.')
log(self, echo=True)
return 0
def __init__(self, *fixedargs, **options):
"Must not throw an exception -- object must always get created."
super(AtsTest, self).__init__()
AtsTest.serialNumber += 1
AtsTest.waitUntilAccumulator.append(self)
# populate attributes
self.serialNumber = AtsTest.serialNumber
if AtsTest.group is None:
AtsTest.groupCounter += 1
self.group = AtsTestGroup(AtsTest.groupCounter)
else:
self.group = AtsTest.group
self.group.append(self)
self.groupNumber = self.group.number
self.groupSerialNumber = len(self.group)
self.waitUntil = AtsTest.waitUntil #never modify this, it may be shared.
self.runOrder = 0 # to aid in diagnosis of wait, priority
self.depends_on = None
self.dependents = []
self.expectedResult = PASSED
self.setName("uninitialized")
self.set(INVALID, "New test, unitialized")
self.srunRelativeNode = -1
self.numNodesToUse = -1
self.priority = -1
self.totalPriority = -1
self.startDateTime = curDateTime()
self.endDateTime = curDateTime()
self.output = [] #magic output, newlines and magic removed.
self.notes = [] #note from the run
self.block = ''
# these will all get changed below but want them set to something for getResults
self.level = 0
self.independent = False
self.np = 1
self.priority = 1
self.totalPriority = 1
self.directory = ''
self.batch = False
self.clas = ''
self.combineOutput = False
self.outname = ''
self.shortoutname = ''
self.errname = ''
self.outhandle = None
self.errhandle = None
self.commandList = ['not run']
# this is just used for documentation
self.commandLine = 'not run'
rootdict = dict(ATSROOT=configuration.ATSROOT)
# Combine the options: first the defaults, then the glued, then the tacked,
# then the stuck, then the test options.
self.options = AttributeDict(
script='',
clas=[],
executable='',
directory='',
)
try:
self.options.update(configuration.options.testDefaults)
self.options.update(AtsTest.glued)
self.options.update(AtsTest.tacked)
self.options.update(AtsTest.stuck)
self.options.update(AtsTest.grouped)
self.options.update(options)
except Exception as e:
self.set(INVALID, 'Bad options: ' + e)
return
self.level = self.options['level']
self.np = self.options['np']
self.priority = self.options.get('priority', max(1, self.np))
self.totalPriority = self.priority
self.testStdout = self.options['testStdout']
outOpts = ['file', 'terminal', 'both']
if not self.testStdout in outOpts:
msg = 'Invalid setting for option testStdout: ' + self.testStdout
raise AtsError(msg)
if configuration.options.allInteractive:
self.batch = False
else:
self.batch = self.options['batch']
if configuration.options.combineOutErr:
self.combineOutput = True
else:
self.combineOutput = False
# process the arguments
# Note: old interface was script, clas='', **options
# Now allow for possibility of no script, or clas as unnamed second
# positional
lc = len(fixedargs)
if lc > 2:
self.set(INVALID, 'Too many positional arguments to test command.')
return
elif lc == 2:
self.options['script'] = fixedargs[0]
self.options['clas'] = fixedargs[1]
elif lc == 1:
self.options['script'] = fixedargs[0]
script = self.options['script']
clas = self.options['clas']
if isinstance(clas, str):
clas = configuration.machine.split(clas)
self.clas = [c % self.options for c in clas]
executable = str(self.options.get('executable'))
self.directory = self.options['directory']
if executable == '1':
if not script:
self.set(INVALID, "executable = 1 requires a first argument.")
return
script = script.replace('$ATSROOT', configuration.ATSROOT)
if len(configuration.ATSROOT) == 0:
script = script[1:] # remove leading "/" or "\"
script = script % rootdict
self.executable = Executable(script)
if self.directory == '':
self.directory = os.getcwd()
path = self.executable.path
junk, filename = os.path.split(path)
else:
if executable:
executable = executable.replace('$ATSROOT',
configuration.ATSROOT)
self.executable = Executable(executable % rootdict)
else:
self.executable = configuration.defaultExecutable
if script:
script = abspath(script) % self.options
self.clas.insert(0, script)
if self.directory == '':
self.directory, filename = os.path.split(script)
else:
if self.directory == '':
self.directory = os.getcwd()
junk, filename = os.path.split(self.executable.path)
name, junk = os.path.splitext(filename)
self.setName(self.options.get('name', name))
label = self.options.get('label', '')
if label:
label = str(label).strip()
self.setName(self.name + '(' + label + ')')
if debug():
log("Results of parsing test arguments", echo=False)
log.indent()
log("Name:", self.name, echo=False)
log("Options:", echo=False)
log.indent()
for k in self.options:
log(k, ": ", self.options[k], echo=False)
log.dedent()
log("Executable path:", self.executable.path, echo=False)
log("Directory:", self.directory, echo=False)
log.dedent()
self.independent = self.options.get('independent', False)
if not self.independent:
# the lower() is due to peculiarities on at least the Mac
# where os.chdir() seems to change case partially.
self.block = self.directory.lower()
if not self.executable.is_valid():
self.set(INVALID, 'Executable "%s" not valid.' % self.executable)
return
if not os.path.isdir(self.directory):
self.set(INVALID, 'Directory not valid: %s' % self.directory)
if script and not is_valid_file(script):
self.set(INVALID, "Script %s does not exist." % script)
return
self.fileOutNamesSet()
#set the timelimit
try:
tl = options.get('timelimit', None)
if tl is None:
self.timelimit = configuration.timelimit
else:
self.timelimit = Duration(tl)
except AtsError as msg:
self.set(INVALID, msg)
return
if self.priority <= 0:
self.set(SKIPPED, 'Test has priority <= zero.')
return
# if the test ends up BATCHED, such jobs are legal.
if self.batch and configuration.options.nobatch:
self.set(SKIPPED, "Batch not available")
elif self.batch:
problem = configuration.batchmachine.canRun(self)
if not problem:
if configuration.options.skip:
self.set(SKIPPED, "BACH skipped due to skip flag")
else:
self.set(BATCHED, "Ready to run in batch.")
else:
self.set(SKIPPED, problem)
else:
problem = configuration.machine.canRun(self)
if not problem:
self.set(CREATED, "Ready to run interactively.")
elif configuration.options.allInteractive or \
configuration.options.nobatch or \
self.groupNumber:
self.set(SKIPPED, problem)
else:
self.set(BATCHED, problem)
self.notes.append(\
"Changed to batch since unable to run interactively on this machine.")
def _launch(self, test):
"""Replace if not using subprocess (unusual).
The subprocess part of launch. Also the part that might fail.
"""
if MachineCore.debugClass:
print("DEBUG MachineCore._launch invoked cwd= %s " % os.getcwd())
#print self
#print test
#print test.options
#print test.__dict__
#print self.__dict__
from ats import configuration
# See if user specified a file to use as stdin to the test problem.
stdin_file = test.options.get('stdin', None)
globalPrerunScript_outname = test.globalPrerunScript_outname
globalPrerunScript = test.options.get('globalPrerunScript', None)
#verbose = test.options.get('verbose', False)
verbose = configuration.options.debug
if not (globalPrerunScript == "unset"):
here = os.getcwd()
os.chdir(test.directory)
if os.path.exists(globalPrerunScript):
self._executePreOrPostRunScript(globalPrerunScript, test,
verbose,
globalPrerunScript_outname)
else:
log("ERROR: globalPrerunScript %s not found" %
(globalPrerunScript),
echo=True)
sys.exit(-1)
os.chdir(here)
try:
Eadd = test.options.get('env', None)
if Eadd is None:
E = None
else:
# This is old Paul DuBois coding, with ugly syntax.
#
# That is the syntax for the user within a 'test' or #ATS line is:
# env={'ANIMAL': 'duck', 'CITY': 'Seattle', 'PLANET': 'Venus'}
#
# The apparent reason for this ugliness, is that is how the environment
# object is stored in Python. So it amakes the coding easier
# here, but it shifts the burden to the user to get that syntax correct,
# including the brackets, quotes, commans, and colons.
#
# Will live with this for now, but would really like this to be more human friendly
#
# env="ANIMAL=duck, CITY=Seattle, PLANET=Venus"
#
if MachineCore.debugClass:
print("DEBUG MachineCore._launch env specified = %s " %
Eadd)
E = os.environ.copy()
E.update(Eadd)
testStdout = test.stdOutLocGet()
if stdin_file is None:
#print "DEBUG MachineCore._launch 010 "
testStdin = None
else:
#print "DEBUG MachineCore._launch 020 "
testStdin = open(test.directory + '/' + stdin_file)
# 2016-09-01
# Starting jobs too fast confuses slurm and MPI. Short wait between each job submittal
# This showsd up with my atsHello test program
# Default sleep is 1 on toss, 0 on other systems, may be set by user on command line
# 2016-12-02
# Default sleep is now 0 on all systems.
if hasattr(test, 'runningWithinSalloc'):
if configuration.options.sleepBeforeSrun > 0:
if MachineCore.printSleepBeforeSrunNotice:
MachineCore.printSleepBeforeSrunNotice = False
print(
"ATS Info: MachineCore._launch Will sleep %d seconds before each srun "
% configuration.options.sleepBeforeSrun)
time.sleep(configuration.options.sleepBeforeSrun)
else:
if configuration.options.sleepBeforeSrun > 0:
if MachineCore.printSleepBeforeSrunNotice:
MachineCore.printSleepBeforeSrunNotice = False
print(
"ATS Info: MachineCore._launch Will sleep %d seconds before each srun "
% configuration.options.sleepBeforeSrun)
time.sleep(configuration.options.sleepBeforeSrun)
if testStdout == 'file':
# Get the file handles for standard out and standard error
outhandle, errhandle = test.fileHandleGet()
# Prepend information about the test to its standard output
self.log_prepend(test, test.outhandle)
if stdin_file is None:
# SAD ambyr
#print "DEBUG MachineCore._launch 050 "
#print "DEBUG MachineCore._launch %s " % test.commandList
#print E
#test.child = subprocess.Popen(test.commandList, cwd=test.directory, stdout=outhandle, stderr=errhandle, env=E, text=True)
test.child = subprocess.Popen(test.commandList,
universal_newlines=True,
cwd=test.directory,
stdout=outhandle,
stderr=errhandle,
env=E,
text=True)
#test.child.wait()
else:
#print "DEBUG MachineCore._launch 110 "
#print testStdin
#print "DEBUG MachineCore._launch 120 test.directory = %s" % test.directory
test.child = subprocess.Popen(test.commandList,
cwd=test.directory,
stdout=outhandle,
stderr=errhandle,
env=E,
stdin=testStdin,
text=True)
elif testStdout == 'terminal':
if MachineCore.debugClass:
print("DEBUG MachineCore._launch Invoking Popen 2 %s " %
test.commandList)
if stdin_file is None:
test.child = subprocess.Popen(test.commandList,
cwd=test.directory,
env=E,
text=True)
else:
test.child = subprocess.Popen(test.commandList,
cwd=test.directory,
env=E,
stdin=testStdin,
text=True)
elif testStdout == 'both':
# Get the file handles for standard out and standard error
outhandle, errhandle = test.fileHandleGet()
# Prepend information about the test to its standard output
self.log_prepend(test, test.outhandle)
if MachineCore.debugClass:
print("DEBUG MachineCore._launch Invoking Popen 3 %s " %
test.commandList)
if stdin_file is None:
test.child = subprocess.Popen(test.commandList,
cwd=test.directory,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=E,
text=True)
else:
test.child = subprocess.Popen(test.commandList,
cwd=test.directory,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=E,
stdin=testStdin)
test.set(RUNNING, test.commandLine)
self.running.append(test)
self.numberTestsRunning += 1
if MachineCore.debugClass or MachineCore.canRunNow_debugClass:
print(
"DEBUG MachineCore.testEnded increased self.numberTestsRunning by 1 to %d "
% self.numberTestsRunning)
if test.numNodesToUse > 0:
self.numberNodesExclusivelyUsed += test.numNodesToUse
if MachineCore.debugClass or MachineCore.canRunNow_debugClass:
print(
"DEBUG MachineCore._launch__ increased self.numberNodesExclusivelyUsed by %d to %d (max is %d)"
% (test.numNodesToUse, self.numberNodesExclusivelyUsed,
self.numNodes))
return True
except OSError as e:
if test.stdOutLocGet() != 'terminal':
test.fileHandleClose()
test.set(FAILED, str(e))
return False
def firstBanner(self):
"Write the opening banner."
log.echo = True
log('ATS START', atsStartTimeLong)
log('ATS VERSION', version.version)
log('ATS HOST NAME:', socket.gethostname())
log('ATS LOG DIRECTORY:', log.directory)
log('SYS_TYPE:', configuration.SYS_TYPE)
log('MACHINE_TYPE', configuration.MACHINE_TYPE)
log('BATCH_TYPE', configuration.BATCH_TYPE)
log("Machine description: ", self.machine.label(), echo=True)
if self.batchmachine:
log("Batch facility name:", self.batchmachine.label(), echo=True)
else:
log("No batch facility found.", echo=True)
if not configuration.options.logUsage:
log('NOT logging usage.')
if configuration.options.info or debug():
configuration.documentConfiguration()
log.echo = self.verbose
if configuration.options.oneFailure:
log('Will stop after first failure.')
if configuration.options.allInteractive:
log('Will run all tests (including any batch tests) as interactive.'
)
log('Default time limit for each test=',
Duration(configuration.timelimit))
def load(self, batchlist):
"Simulate the batch system"
log("Simulation of batch load:", echo=True)
for t in batchlist:
log(t, echo=True)
def init(clas = '', adder = None, examiner=None):
"""Called by manager.init(class, adder, examiner)
Initialize configuration and process command-line options; create log,
options, inputFiles, timelimit, machine, and batchmatchine.
Call backs to machine and to adder/examiner for options.
"""
global log, options, inputFiles, timelimit, machine, batchmachine,\
defaultExecutable, ATSROOT, cuttime
init_debugClass = False
if init_debugClass:
print("DEBUG init entered clas=%s " % (clas))
# get the machine and possible batch facility
machineDirs = MACHINE_DIR
# delete, not needed, handled above
#if MACHINE_OVERRIDE_DIR:
# machineDirs.append(MACHINE_OVERRIDE_DIR)
machineList = []
for machineDir in machineDirs:
log('machineDir', machineDir)
machineList.extend([os.path.join(machineDir,x) for x in os.listdir(machineDir) if x.endswith('.py') and not x.endswith('__init__.py')])
sys.path.insert(0, machineDir)
#machineList = []
#for machineDir in machineDirs:
# print("DEBUG machineDir=%s " % (machineDir))
# machineList.extend(
# [os.path.join(machineDir,x) for x in os.listdir(machineDir)
# if x.endswith('.py') and not x.endswith('__init__.py')])
machine = None
batchmachine = None
specFoundIn = ''
bspecFoundIn = ''
if init_debugClass:
print("DEBUG init 100")
print(machineDirs)
print(machineList)
print(MACHINE_TYPE)
print("DEBUG init 200")
for full_path in machineList:
moduleName = ''
fname = os.path.basename(full_path)
# print "DEBUG 000 fname = %s" % fname
f = open(full_path, 'r')
for line in f:
if line.startswith('#ATS:') and not machine:
items = line[5:-1].split()
machineName, moduleName, machineClass, npMaxH = items
if init_debugClass:
print("DEBUG init machineName=%s moduleName=%s machineClass=%s npMaxH=%s" %
(machineName, moduleName, machineClass, npMaxH))
# print "DEBUG init MACHINE_TYPE=%s machineName=%s moduleName=%s machineClass=%s npMaxH=%s" % (MACHINE_TYPE, machineName, moduleName, machineClass, npMaxH)
if machineName == MACHINE_TYPE:
if moduleName == "SELF":
moduleName, junk = os.path.splitext(fname)
specFoundIn = full_path
print(f"from ats.atsMachines.{moduleName} "
f"import {machineClass} as Machine")
try:
machine_factory = get_machine_factory(moduleName,
machineClass)
except ModuleNotFoundError:
machine_factory = get_machine_factory(moduleName,
machineClass,
machine_package='atsMachines')
machine = machine_factory(machineName, int(npMaxH))
elif line.startswith('#BATS:') and not batchmachine:
items = line[6:-1].split()
machineName, moduleName, machineClass, npMaxH = items
if machineName == BATCH_TYPE:
if moduleName == "SELF":
moduleName, junk = os.path.splitext(fname)
bspecFoundIn = full_path
try:
machine_factory = get_machine_factory(moduleName,
machineClass)
except ModuleNotFoundError:
machine_factory = get_machine_factory(moduleName,
machineClass,
machine_package='atsMachines')
batchmachine = machine_factory(moduleName, int(npMaxH))
f.close()
if machine and batchmachine:
break
if machine is None:
terminal("No machine specifications for", SYS_TYPE, "found, using generic.")
machine = machines.Machine('generic', -1)
# create the option set
usage = "usage: %prog [options] [input files]"
parser = OptionParser(usage=usage, version="%prog " + version.version)
addOptions(parser)
machine.addOptions(parser)
# add the --nobatch option but force it true if no batch facility here.
parser.add_option('--nobatch', action='store_true', dest='nobatch', default=(batchmachine is None),
help = 'Do not run batch jobs.')
if batchmachine:
batchmachine.addOptions(parser)
# user callback?
if adder is not None:
adder(parser)
# parse the command line
if clas:
import shlex
argv = shlex.split(clas)
else:
argv = sys.argv[1:]
(toptions, inputFiles) = parser.parse_args(argv)
# immediately make the options a real dictionary -- the way optparse leaves it
# is misleading.
options = AttributeDict()
for k in vars(toptions).keys():
options[k] = getattr(toptions, k)
# set up the test default options so the machine(s) can add to it
options['testDefaults'] = AttributeDict(np=1,
batch=0,
level=1,
keep = options.keep,
hideOutput = options.hideOutput,
verbose = options.verbose,
testStdout = options.testStdout,
globalPrerunScript = options.globalPrerunScript,
globalPostrunScript = options.globalPostrunScript,
sequential = options.sequential,
nosrun = options.nosrun,
salloc = options.salloc
)
# let the machine(s) modify the results or act upon them in other ways.
machine.examineOptions(options)
if batchmachine:
batchmachine.examineOptions(options)
# unpack basic options
debug(options.debug)
if options.logdir:
log.set(directory = options.logdir)
else:
dirname = SYS_TYPE + "." + atsStartTime + ".logs"
log.set(directory = dirname)
log.mode="w"
log.logging = 1
# user callback?
if examiner is not None:
examiner(options)
if specFoundIn:
log("Found specification for", MACHINE_TYPE, "in", specFoundIn)
else:
log("No specification found for", MACHINE_TYPE, ', using generic')
if bspecFoundIn:
log("Batch specification for ", BATCH_TYPE, "in", bspecFoundIn)
# unpack other options
cuttime = options.cuttime
if cuttime is not None:
cuttime = Duration(cuttime)
timelimit = Duration(options.timelimit)
defaultExecutable = executables.Executable(abspath(options.executable))
# ATSROOT is used in tests.py to allow paths pointed at the executable's directory
if 'ATSROOT' in os.environ:
ATSROOT = os.environ['ATSROOT']
else:
ATSROOT = os.path.dirname(defaultExecutable.path)
def documentConfiguration():
"""Write the configuration to the log."""
log('Configuration:')
log.indent()
log('Input files:', inputFiles)
log('Python path:', sys.executable)
log('ATS from ', os.path.dirname(__file__))
log('ATS version:', version.version)
log('Options:')
log.indent()
olist = list(options.keys())
olist.sort()
for k in olist:
log(k + ":", repr(getattr(options, k)))
log.dedent()
log.dedent()
def core(self):
"This is the 'guts' of ATS."
if configuration.SYS_TYPE == "toss_3_x86_64":
if configuration.options.bypassSerialMachineCheck == False:
log("**********************************************************************************",
echo=True)
log("*** This is a serial machine --- Do not use ATS on more than 1 node here! ***",
echo=True)
log("*** ***",
echo=True)
log("*** Use ATS option --bypassSerialMachineCheck if you promise to run on 1 Node ***",
echo=True)
log("**********************************************************************************",
echo=True)
sys.exit(-1)
# Phase 1 -- collect the tests
errorOccurred = False
try: # surround with keyboard interrupt, AtsError handlers
self.collectTests()
except AtsError:
log("ATS error while collecting tests.", echo=True)
log(traceback.format_exc(), echo=True)
errorOccurred = True
self.collectTimeEnded = datestamp(long_format=True)
except KeyboardInterrupt:
log("Keyboard interrupt while collecting tests, terminating.",
echo=True)
errorOccurred = True
self.collectTimeEnded = datestamp(long_format=True)
if errorOccurred:
return
try:
for f in self.onCollectedRoutines:
log("Calling onCollected routine",
f.__name__,
echo=self.verbose)
f(self)
except KeyboardInterrupt:
log("Keyboard interrupt while collecting tests, terminating.",
echo=True)
errorOccurred = True
except Exception:
log("Error in user-specified onCollected routine.", echo=True)
log(traceback.format_exc(), echo=True)
errorOccured = True
if errorOccurred:
return
# divide into interactive and batch tests
interactiveTests, batchTests = self.sortTests()
if len(interactiveTests) + len(batchTests) == 0:
log("No tests found.", echo=True)
return
# We have built up the list of tests. Run functions added via
# beforeRun() calls to allow user to do stuff like cleaning up old test
# results before running or other things.
self.preprocess()
# Phase 2 -- dispatch the batch tests
if self.batchmachine and batchTests:
if configuration.options.skip:
log("Skipping execution due to --skip")
else:
try:
log("Sending %d tests to %s." %
(len(batchTests), self.batchmachine.name),
echo=True)
self.batchmachine.load(batchTests)
except AtsError:
log(traceback.format_exc(), echo=True)
log("ATS error.", echo=True)
return
except KeyboardInterrupt:
log("Keyboard interrupt while dispatching batch, terminating.",
echo=True)
return
# Phase 3 -- run the interactive tests
dieDieDie = False
if interactiveTests:
self.machine.scheduler.prioritize(interactiveTests)
try:
log("Total number of interactive tests = ",
len(interactiveTests),
echo=True)
log("---------------------------------------------------",
echo=True)
for f in self.onPrioritizedRoutines:
log("Calling onPrioritized routine",
f.__name__,
echo=self.verbose)
f(interactiveTests)
except KeyboardInterrupt:
log("Keyboard interrupt while prioritizing tests, terminating.",
echo=True)
errorOccurred = True
except Exception:
log("Error in prioritizing tests.", echo=True)
log(traceback.format_exc(), echo=True)
errorOccured = True
if errorOccurred:
return
try:
self.run(interactiveTests)
except AtsError:
log(traceback.format_exc(), echo=True)
log("ATS error. Removing running jobs....", echo=True)
dieDieDie = True
except KeyboardInterrupt:
dieDieDie = True
log("Keyboard interrupt. Removing running jobs....", echo=True)
if dieDieDie:
time.sleep(3)
for test in self.testlist:
if (test.status is RUNNING):
self.machine.kill(test)
self.machine.quit() #machine shutdown / cleanup
# Phase 4 -- Continuation file
# for t in interactiveTests:
# if t.status not in (PASSED, EXPECTED, FILTERED):
# break
# else:
# self.continuationFileName = ''
# return
self.continuationFile(interactiveTests)
def launch(self, test):
"""Start executable using a suitable command.
Return True if able to do so.
Call noteLaunch if launch succeeded."""
from ats import configuration
#print test.__dict__
##print self.__dict__
nosrun = test.options.get('nosrun', False)
serial = test.options.get(
'serial', False
) # support serial=True on a per-test basis for backwards compatability for a while
if nosrun == True or serial == True:
test.commandList = self.calculateBasicCommandList(test)
test.cpus_per_task = 1
else:
test.commandList = self.calculateCommandList(test)
if test.commandList == None:
log("ATS def launch returning false, commandList is None",
echo=True)
return False
#
# On Blueos (Sierra/Ansel) Set JSM_JSRUN_NO_WARN_OVERSUBSCRIBE the same as lrun does
#
if configuration.SYS_TYPE.startswith('blueos'):
os.environ['JSM_JSRUN_NO_WARN_OVERSUBSCRIBE'] = '1'
# To enable running of threaded codes in 1 thread mode, the OMP_NUM_THREADS must be
# set either by the user before the run, or by the test 'nt' option, or by
# the command line option to ATS --ompNumThreads. If none of these are set, then
# set it to 1.
if configuration.options.ompNumThreads > 0:
# Priority 1 setting, ats command line
if configuration.options.verbose:
print(
"ATS launch setting OMP_NUM_THREADS %d as user specified --ompNumThreads=%d"
% (configuration.options.ompNumThreads,
configuration.options.ompNumThreads))
os.environ['OMP_NUM_THREADS'] = str(
configuration.options.ompNumThreads)
else:
# Priority 2 setting, within an ATS test line
omp_num_threads = test.options.get('nt', -1)
if (omp_num_threads > 0):
if configuration.options.verbose:
print(
"ATS launch setting OMP_NUM_THREADS %d based on test 'nt'option"
% omp_num_threads)
os.environ['OMP_NUM_THREADS'] = str(omp_num_threads)
else:
# Priority 3 setting, the user has already set OMP_NUM_THREADS in their environment
if 'OMP_NUM_THREADS' in os.environ:
if configuration.options.verbose:
temp_omp = os.getenv("OMP_NUM_THREADS")
# print "ATS detected that OMP_NUM_THREADS is already set to %s" % (temp_omp)
# Priority 4 setting, set it to 1 if it is not othewise set
else:
if configuration.options.verbose:
print(
"ATS launch setting OMP_NUM_THREADS 1 by default for as it was not specified for the test."
)
# print " This should allow for threaded applications to run with non threaded tests with a single thread."
os.environ['OMP_NUM_THREADS'] = str(1)
# Set default KMP_AFFINITY so that OpenMP runs are OK on Toss 3
# This is experimental for now.
if configuration.SYS_TYPE.startswith('toss'):
if MachineCore.printExperimentalNotice:
MachineCore.printExperimentalNotice = False
print("ATS Experimental: setting KMP_AFFINITY to %s on Toss" %
configuration.options.kmpAffinity)
os.environ['KMP_AFFINITY'] = configuration.options.kmpAffinity
# Turn off shared memory mpi collective operations on toss and chaos
if configuration.SYS_TYPE.startswith('toss'):
os.environ['VIADEV_USE_SHMEM_COLL'] = "0"
if configuration.SYS_TYPE.startswith('chaos'):
os.environ['VIADEV_USE_SHMEM_COLL'] = "0"
# LS_COLORS can mess up somesystem and is not needed for any platform by ATS
os.environ['LS_COLORS'] = ""
# Bamboo env vars can also mess up somesystem runs by exceeding char limit for env vars
# remove them
os.environ['bamboo_shortJobName'] = ""
os.environ['bamboo_capability_system_git_executable'] = ""
os.environ['bamboo_build_working_directory'] = ""
os.environ['bamboo_shortPlanKey'] = ""
os.environ['bamboo_planName'] = ""
os.environ['bamboo_capability_system_jdk_JDK_1_8_0_71'] = ""
os.environ['bamboo_buildKey'] = ""
os.environ['bamboo_capability_system_jdk_JDK'] = ""
os.environ['bamboo_capability_sys_type'] = ""
os.environ['bamboo_capability_cluster'] = ""
os.environ['bamboo_buildFailed'] = ""
os.environ['bamboo_buildResultKey'] = ""
os.environ['bamboo_plan_storageTag'] = ""
os.environ['bamboo_planKey'] = ""
os.environ['bamboo_capability_system_builder_ant_Ant'] = ""
os.environ['bamboo_shortPlanName'] = ""
os.environ['bamboo_buildResultsUrl'] = ""
os.environ['bamboo_buildPlanName'] = ""
os.environ['bamboo_working_directory'] = ""
os.environ['bamboo_agentWorkingDirectory'] = ""
os.environ['bamboo_buildTimeStamp'] = ""
os.environ['bamboo_shortJobKey'] = ""
os.environ['bamboo_buildNumber'] = ""
os.environ['bamboo_agentId'] = ""
os.environ['bamboo_capability_system_jdk_JDK_1_8'] = ""
os.environ['bamboo_resultsUrl'] = ""
test.commandLine = " ".join(test.commandList)
sandbox = test.options.get('sandbox', False)
directory = test.options.get('directory', None)
deck_directory = test.options.get('deck_directory', None)
if sandbox:
# directory is the name of the sandbox directory
if directory == None or directory == '':
directory = ('%s_%d_%04d_%s') % (
'sandbox', os.getpid(), test.serialNumber, test.namebase)
if deck_directory == None or deck_directory == '':
deck_directory = os.getcwd()
if not os.path.isdir(directory):
if MachineCore.debugClass:
print(
"MachineCore.launch \n\tcwd=%s \n\tdir=%s \n\tdeck_directory=%s"
% (os.getcwd(), directory, deck_directory))
log("ATS machines.py Creating sandbox directory : %s" %
directory,
echo=True)
copytree(deck_directory,
directory,
ignore=ignore_patterns('*.logs', 'html', '.svn',
'*sandbox*'))
#--- placing this here doesn't allow the machines to handle the skip option themselves..
if configuration.options.skip:
test.set(SKIPPED, "--skip option")
return False
test.setStartDateTime()
result = self._launch(test)
if result:
self.noteLaunch(test)
return result
