def getResults(self):
"""Returns an attribute dictionary containing the state of this
manager suitable for postprocessing. After forming a potential
result r, calls any resultsHooks functions (r, manager)
"""
r = AttributeDict(
started=self.started,
options=self.options,
savedTime=datestamp(long_format=True),
collectTimeEnded=self.collectTimeEnded,
badlist=self.badlist,
filters=self.filters,
groups={},
onCollectedRoutines=[f.__name__ for f in self.onCollectedRoutines],
onPrioritizedRoutines=[
f.__name__ for f in self.onPrioritizedRoutines
],
onExitRoutines=[f.__name__ for f in self.onExitRoutines],
onResultsRoutines=[f.__name__ for f in self.onResultsRoutines],
)
r.testlist = [t.getResults() for t in self.testlist]
if not hasattr(self, 'machine'):
return r # never initialized, nothing else of interest.
r.inputFiles = self.inputFiles
r.verbose = self.verbose
r.machine = AttributeDict()
r.batchmachine = None
for key, value in self.machine.getResults().items():
r.machine[key] = value
if self.batchmachine:
r.batchmachine = AttributeDict()
for key, value in self.batchmachine.getResults().items():
r.batchmachine[key] = value
for hook in self.onResultsRoutines:
log(' Calling onResults function', hook.__name__, echo=True)
hook(r, self)
return r
def getResults(self):
"Return a dictionary containing the essential information about this test."
if not hasattr(self, 'timelimit'):
self.timelimit = None
if self.depends_on is None:
ds = 0
else:
ds = self.depends_on.serialNumber
if self.output:
out = ['Captured output, see log.']
else:
out = []
# various integers are used to reconstruct the test/group relationships
# See the coding in management.py that prints atsr.py
result = AttributeDict(
name=self.name,
serialNumber=self.serialNumber,
groupNumber=self.groupNumber,
groupSerialNumber=self.groupSerialNumber,
runOrder=self.runOrder,
status=self.status,
batch=self.batch,
expectedResult=self.expectedResult,
message=self.message,
startDateTime=self.startDateTime,
endDateTime=self.endDateTime,
options=self.options,
directory=self.directory,
notes=self.notes,
output=out,
independent=self.independent,
block=self.block,
timelimit=self.timelimit,
commandList=self.commandList,
commandLine=self.commandLine,
elapsedTime=self.elapsedTime(),
executable=str(self.executable),
priority=self.priority,
totalPriority=self.totalPriority,
depends_on_serial=ds,
dependents_serial=[d.serialNumber for d in self.dependents],
waitUntil_serial=[t.serialNumber for t in self.waitUntil])
return result
def getResults(self):
"""
A child might replace this to put more information in the results,
but probaby wants to call the parent and then update the
dictionary this method returns.
Return AttributeDict of machine-specific facts for manager postprocessing state.
Include results from the scheduler.
"""
result = AttributeDict()
result.update(self.scheduler.getResults())
result.update(
dict(name=self.name,
numberTestsRunningMax=self.numberTestsRunningMax,
hardLimit=self.hardLimit,
naptime=self.naptime))
return result
def getResults(self):
"""Results for the atsr file."""
return AttributeDict(scheduler=self.name)
def __init__(self, *fixedargs, **options):
"Must not throw an exception -- object must always get created."
super(AtsTest, self).__init__()
AtsTest.serialNumber += 1
AtsTest.waitUntilAccumulator.append(self)
# populate attributes
self.serialNumber = AtsTest.serialNumber
if AtsTest.group is None:
AtsTest.groupCounter += 1
self.group = AtsTestGroup(AtsTest.groupCounter)
else:
self.group = AtsTest.group
self.group.append(self)
self.groupNumber = self.group.number
self.groupSerialNumber = len(self.group)
self.waitUntil = AtsTest.waitUntil #never modify this, it may be shared.
self.runOrder = 0 # to aid in diagnosis of wait, priority
self.depends_on = None
self.dependents = []
self.expectedResult = PASSED
self.setName("uninitialized")
self.set(INVALID, "New test, unitialized")
self.srunRelativeNode = -1
self.numNodesToUse = -1
self.priority = -1
self.totalPriority = -1
self.startDateTime = curDateTime()
self.endDateTime = curDateTime()
self.output = [] #magic output, newlines and magic removed.
self.notes = [] #note from the run
self.block = ''
# these will all get changed below but want them set to something for getResults
self.level = 0
self.independent = False
self.np = 1
self.priority = 1
self.totalPriority = 1
self.directory = ''
self.batch = False
self.clas = ''
self.combineOutput = False
self.outname = ''
self.shortoutname = ''
self.errname = ''
self.outhandle = None
self.errhandle = None
self.commandList = ['not run']
# this is just used for documentation
self.commandLine = 'not run'
rootdict = dict(ATSROOT=configuration.ATSROOT)
# Combine the options: first the defaults, then the glued, then the tacked,
# then the stuck, then the test options.
self.options = AttributeDict(
script='',
clas=[],
executable='',
directory='',
)
try:
self.options.update(configuration.options.testDefaults)
self.options.update(AtsTest.glued)
self.options.update(AtsTest.tacked)
self.options.update(AtsTest.stuck)
self.options.update(AtsTest.grouped)
self.options.update(options)
except Exception as e:
self.set(INVALID, 'Bad options: ' + e)
return
self.level = self.options['level']
self.np = self.options['np']
self.priority = self.options.get('priority', max(1, self.np))
self.totalPriority = self.priority
self.testStdout = self.options['testStdout']
outOpts = ['file', 'terminal', 'both']
if not self.testStdout in outOpts:
msg = 'Invalid setting for option testStdout: ' + self.testStdout
raise AtsError(msg)
if configuration.options.allInteractive:
self.batch = False
else:
self.batch = self.options['batch']
if configuration.options.combineOutErr:
self.combineOutput = True
else:
self.combineOutput = False
# process the arguments
# Note: old interface was script, clas='', **options
# Now allow for possibility of no script, or clas as unnamed second
# positional
lc = len(fixedargs)
if lc > 2:
self.set(INVALID, 'Too many positional arguments to test command.')
return
elif lc == 2:
self.options['script'] = fixedargs[0]
self.options['clas'] = fixedargs[1]
elif lc == 1:
self.options['script'] = fixedargs[0]
script = self.options['script']
clas = self.options['clas']
if isinstance(clas, str):
clas = configuration.machine.split(clas)
self.clas = [c % self.options for c in clas]
executable = str(self.options.get('executable'))
self.directory = self.options['directory']
if executable == '1':
if not script:
self.set(INVALID, "executable = 1 requires a first argument.")
return
script = script.replace('$ATSROOT', configuration.ATSROOT)
if len(configuration.ATSROOT) == 0:
script = script[1:] # remove leading "/" or "\"
script = script % rootdict
self.executable = Executable(script)
if self.directory == '':
self.directory = os.getcwd()
path = self.executable.path
junk, filename = os.path.split(path)
else:
if executable:
executable = executable.replace('$ATSROOT',
configuration.ATSROOT)
self.executable = Executable(executable % rootdict)
else:
self.executable = configuration.defaultExecutable
if script:
script = abspath(script) % self.options
self.clas.insert(0, script)
if self.directory == '':
self.directory, filename = os.path.split(script)
else:
if self.directory == '':
self.directory = os.getcwd()
junk, filename = os.path.split(self.executable.path)
name, junk = os.path.splitext(filename)
self.setName(self.options.get('name', name))
label = self.options.get('label', '')
if label:
label = str(label).strip()
self.setName(self.name + '(' + label + ')')
if debug():
log("Results of parsing test arguments", echo=False)
log.indent()
log("Name:", self.name, echo=False)
log("Options:", echo=False)
log.indent()
for k in self.options:
log(k, ": ", self.options[k], echo=False)
log.dedent()
log("Executable path:", self.executable.path, echo=False)
log("Directory:", self.directory, echo=False)
log.dedent()
self.independent = self.options.get('independent', False)
if not self.independent:
# the lower() is due to peculiarities on at least the Mac
# where os.chdir() seems to change case partially.
self.block = self.directory.lower()
if not self.executable.is_valid():
self.set(INVALID, 'Executable "%s" not valid.' % self.executable)
return
if not os.path.isdir(self.directory):
self.set(INVALID, 'Directory not valid: %s' % self.directory)
if script and not is_valid_file(script):
self.set(INVALID, "Script %s does not exist." % script)
return
self.fileOutNamesSet()
#set the timelimit
try:
tl = options.get('timelimit', None)
if tl is None:
self.timelimit = configuration.timelimit
else:
self.timelimit = Duration(tl)
except AtsError as msg:
self.set(INVALID, msg)
return
if self.priority <= 0:
self.set(SKIPPED, 'Test has priority <= zero.')
return
# if the test ends up BATCHED, such jobs are legal.
if self.batch and configuration.options.nobatch:
self.set(SKIPPED, "Batch not available")
elif self.batch:
problem = configuration.batchmachine.canRun(self)
if not problem:
if configuration.options.skip:
self.set(SKIPPED, "BACH skipped due to skip flag")
else:
self.set(BATCHED, "Ready to run in batch.")
else:
self.set(SKIPPED, problem)
else:
problem = configuration.machine.canRun(self)
if not problem:
self.set(CREATED, "Ready to run interactively.")
elif configuration.options.allInteractive or \
configuration.options.nobatch or \
self.groupNumber:
self.set(SKIPPED, problem)
else:
self.set(BATCHED, problem)
self.notes.append(\
"Changed to batch since unable to run interactively on this machine.")
def getResults(self):
"Return machine-specific facts for manager postprocessing state."
return AttributeDict(name=self.label())
def init(clas = '', adder = None, examiner=None):
"""Called by manager.init(class, adder, examiner)
Initialize configuration and process command-line options; create log,
options, inputFiles, timelimit, machine, and batchmatchine.
Call backs to machine and to adder/examiner for options.
"""
global log, options, inputFiles, timelimit, machine, batchmachine,\
defaultExecutable, ATSROOT, cuttime
init_debugClass = False
if init_debugClass:
print("DEBUG init entered clas=%s " % (clas))
# get the machine and possible batch facility
machineDirs = MACHINE_DIR
# delete, not needed, handled above
#if MACHINE_OVERRIDE_DIR:
# machineDirs.append(MACHINE_OVERRIDE_DIR)
machineList = []
for machineDir in machineDirs:
log('machineDir', machineDir)
machineList.extend([os.path.join(machineDir,x) for x in os.listdir(machineDir) if x.endswith('.py') and not x.endswith('__init__.py')])
sys.path.insert(0, machineDir)
#machineList = []
#for machineDir in machineDirs:
# print("DEBUG machineDir=%s " % (machineDir))
# machineList.extend(
# [os.path.join(machineDir,x) for x in os.listdir(machineDir)
# if x.endswith('.py') and not x.endswith('__init__.py')])
machine = None
batchmachine = None
specFoundIn = ''
bspecFoundIn = ''
if init_debugClass:
print("DEBUG init 100")
print(machineDirs)
print(machineList)
print(MACHINE_TYPE)
print("DEBUG init 200")
for full_path in machineList:
moduleName = ''
fname = os.path.basename(full_path)
# print "DEBUG 000 fname = %s" % fname
f = open(full_path, 'r')
for line in f:
if line.startswith('#ATS:') and not machine:
items = line[5:-1].split()
machineName, moduleName, machineClass, npMaxH = items
if init_debugClass:
print("DEBUG init machineName=%s moduleName=%s machineClass=%s npMaxH=%s" %
(machineName, moduleName, machineClass, npMaxH))
# print "DEBUG init MACHINE_TYPE=%s machineName=%s moduleName=%s machineClass=%s npMaxH=%s" % (MACHINE_TYPE, machineName, moduleName, machineClass, npMaxH)
if machineName == MACHINE_TYPE:
if moduleName == "SELF":
moduleName, junk = os.path.splitext(fname)
specFoundIn = full_path
print(f"from ats.atsMachines.{moduleName} "
f"import {machineClass} as Machine")
try:
machine_factory = get_machine_factory(moduleName,
machineClass)
except ModuleNotFoundError:
machine_factory = get_machine_factory(moduleName,
machineClass,
machine_package='atsMachines')
machine = machine_factory(machineName, int(npMaxH))
elif line.startswith('#BATS:') and not batchmachine:
items = line[6:-1].split()
machineName, moduleName, machineClass, npMaxH = items
if machineName == BATCH_TYPE:
if moduleName == "SELF":
moduleName, junk = os.path.splitext(fname)
bspecFoundIn = full_path
try:
machine_factory = get_machine_factory(moduleName,
machineClass)
except ModuleNotFoundError:
machine_factory = get_machine_factory(moduleName,
machineClass,
machine_package='atsMachines')
batchmachine = machine_factory(moduleName, int(npMaxH))
f.close()
if machine and batchmachine:
break
if machine is None:
terminal("No machine specifications for", SYS_TYPE, "found, using generic.")
machine = machines.Machine('generic', -1)
# create the option set
usage = "usage: %prog [options] [input files]"
parser = OptionParser(usage=usage, version="%prog " + version.version)
addOptions(parser)
machine.addOptions(parser)
# add the --nobatch option but force it true if no batch facility here.
parser.add_option('--nobatch', action='store_true', dest='nobatch', default=(batchmachine is None),
help = 'Do not run batch jobs.')
if batchmachine:
batchmachine.addOptions(parser)
# user callback?
if adder is not None:
adder(parser)
# parse the command line
if clas:
import shlex
argv = shlex.split(clas)
else:
argv = sys.argv[1:]
(toptions, inputFiles) = parser.parse_args(argv)
# immediately make the options a real dictionary -- the way optparse leaves it
# is misleading.
options = AttributeDict()
for k in vars(toptions).keys():
options[k] = getattr(toptions, k)
# set up the test default options so the machine(s) can add to it
options['testDefaults'] = AttributeDict(np=1,
batch=0,
level=1,
keep = options.keep,
hideOutput = options.hideOutput,
verbose = options.verbose,
testStdout = options.testStdout,
globalPrerunScript = options.globalPrerunScript,
globalPostrunScript = options.globalPostrunScript,
sequential = options.sequential,
nosrun = options.nosrun,
salloc = options.salloc
)
# let the machine(s) modify the results or act upon them in other ways.
machine.examineOptions(options)
if batchmachine:
batchmachine.examineOptions(options)
# unpack basic options
debug(options.debug)
if options.logdir:
log.set(directory = options.logdir)
else:
dirname = SYS_TYPE + "." + atsStartTime + ".logs"
log.set(directory = dirname)
log.mode="w"
log.logging = 1
# user callback?
if examiner is not None:
examiner(options)
if specFoundIn:
log("Found specification for", MACHINE_TYPE, "in", specFoundIn)
else:
log("No specification found for", MACHINE_TYPE, ', using generic')
if bspecFoundIn:
log("Batch specification for ", BATCH_TYPE, "in", bspecFoundIn)
# unpack other options
cuttime = options.cuttime
if cuttime is not None:
cuttime = Duration(cuttime)
timelimit = Duration(options.timelimit)
defaultExecutable = executables.Executable(abspath(options.executable))
# ATSROOT is used in tests.py to allow paths pointed at the executable's directory
if 'ATSROOT' in os.environ:
ATSROOT = os.environ['ATSROOT']
else:
ATSROOT = os.path.dirname(defaultExecutable.path)
testEnvironment = AttributeDict(
debug=debug,
manager=manager,
test=test,
testif=testif,
source=source,
log=log,
define=define,
undefine=undefine,
get=get,
logDefinitions=logDefinitions,
filter=filter,
filterdefs=filterdefs,
wait=wait,
stick=stick,
unstick=unstick,
tack=tack,
untack=untack,
group=group,
endgroup=endgroup,
glue=glue,
unglue=unglue,
checkGlue=checkGlue,
getOptions=getOptions,
sys=sys,
os=os,
abspath=abspath,
AtsError=AtsError,
is_valid_file=is_valid_file,
SYS_TYPE=configuration.SYS_TYPE,
MACHINE_TYPE=configuration.MACHINE_TYPE,
BATCH_TYPE=configuration.BATCH_TYPE,
MACHINE_DIR=configuration.MACHINE_DIR,
onCollected=onCollected,
onPrioritized=onPrioritized,
onExit=onExit,
onSave=onSave,
getResults=getResults,
)