def get(self, name):
"""Return the definition of name from the test environment.
"""
if name in testEnvironment:
return testEnvironment.get(name)
raise AtsError("Could not find name %s in vocabulary." % name)
def filterdefs(text=None):
"""Add the given text into the environment
used for filtering.
With no arguments, clear defined list.
"""
global _filterwith
if text is None:
log('filterdefs: erasing definitions')
_filterwith = []
else:
try:
d = {}
for f in _filterwith:
exec(f, d)
exec(text, d)
except SyntaxError as e:
raise AtsError(e)
except KeyboardInterrupt:
raise
except Exception as e:
pass
if debug():
log('filterdefs:')
log.indent()
log(text)
log.dedent()
_filterwith.append(text)
def filterenv(self, test):
"""Compute the environment in which filters for test will be
evaluated."""
if not isinstance(test, AtsTest):
raise AtsError('filterenv argument must be a test instance.')
fe = {}
for f in _filterwith:
exec(f, fe)
fe.update(test.options)
fe.update(testEnvironment)
return fe
def update_test_status(json_response, arg, errnum):
#print >>sys.stderr, "RECEIVED RESPONSE"
#print >>sys.stderr, "JSON", json_response
response = json.loads(json_response)
#print >>sys.stderr, response, jsc.job_num2state(response['state-pair']['nstate'])
test_to_update = arg.submitted.get(response['jobid'], None)
if test_to_update is None:
print("GOT UNEXPECTED RESPONSE %s" % response)
return
new_state = response['state-pair']['nstate']
if new_state >= jsc.lib.J_NULL and new_state < jsc.lib.J_RUNNING:
test_to_update.fluxStatus = 'submitted'
test_to_update.set(RUNNING, "Submitted, pending allocation"
) #not really true... but as close as they come
elif new_state == jsc.lib.J_RUNNING:
if test_to_update.fluxStatus != 'running':
arg.running.append(test_to_update)
with kvs.get_dir(arg.fh, test_to_update.kvs_path) as d:
test_to_update.startTime = float(d['running-time'])
test_to_update.fluxStatus = 'running'
test_to_update.set(RUNNING, test_to_update.commandLine)
elif new_state > jsc.lib.J_RUNNING and new_state != jsc.lib.J_COMPLETING:
if test_to_update.fluxStatus != 'done':
arg.running.remove(test_to_update)
arg.submitted.pop(test_to_update.job_id, None)
test_to_update.fluxStatus = 'done'
status = HALTED # test is done, pre-set to a did-not-run state # TODO: see if this is the right code for this
if new_state == jsc.lib.J_COMPLETE:
# flux says it ran ok, check return code
with kvs.get_dir(arg.fh, test_to_update.kvs_path) as d:
try:
exit_status = d['exit_status']
except:
exit_status = 5
for k in d:
print("LWJ KVS DEBUG %s=%s" % (k, d[k]))
test_to_update.endTime = float(d['complete-time'])
if exit_status['min'] == exit_status['max'] and exit_status[
'min'] == 0:
status = PASSED
else:
status = FAILED
else:
# it didn't run ok, don't check anything else
if configuration.options.oneFailure:
raise AtsError("Test failed in oneFailure mode.")
print("UPDATING TEST STATUS TO %s" % status, file=sys.stderr)
test_to_update.set(status, test_to_update.elapsedTime())
arg.noteEnd(test_to_update)
def filter(self, *filters):
"Add filters to the list. Clear list if no arguments."
if not filters:
self.filters = []
log('Filter list empty.', echo=self.verbose)
return
for f in filters:
try:
f = str(f)
except Exception:
raise AtsError("filter must be convertible to string")
if not f:
continue
try:
r = eval(f, {}, {})
except SyntaxError:
raise AtsError('Mal-formed filter, %s' % repr(f))
except KeyboardInterrupt:
raise
except Exception:
pass
self.filters.append(f)
log('Added filter:', repr(f))
def checkRunning(self):
"""Find those tests still running. getStatus checks for timeout.
"""
# print "DEBUG checkRunning 100\n"
from ats import configuration
time.sleep(self.naptime)
stillRunning = []
for test in self.running:
done = self.getStatus(test)
if not done:
stillRunning.append(test)
else: # test has finished
if test.status is not PASSED:
if configuration.options.oneFailure:
raise AtsError("Test failed in oneFailure mode.")
self.running = stillRunning
def timeSpecToSec(spec):
"Return minutes from an integer or string spec of the form nn h nn m nn s"
specIn = str(spec)
spec = str(spec).strip().lower()
posH = spec.find('h')
posM = spec.find('m')
posS = spec.find('s')
if posH == -1 and posM == -1 and posS == -1:
spec = spec + 'm'
posM = spec.find('m')
try:
if posH > 0:
h = int(spec[0:posH])
else:
h = 0
if posM > posH:
m = int(spec[posH + 1:posM])
else:
m = 0
posM = posH
if posS > posM:
s = int(spec[posM + 1:posS])
else:
sp = spec[posM + 1:]
if not sp:
s = 0
else:
s = int(sp)
return (h * 3600 + 60 * m + s)
except Exception as e:
raise AtsError("Bad time specification: %s" % specIn)
def __init__(self, *fixedargs, **options):
"Must not throw an exception -- object must always get created."
super(AtsTest, self).__init__()
AtsTest.serialNumber += 1
AtsTest.waitUntilAccumulator.append(self)
# populate attributes
self.serialNumber = AtsTest.serialNumber
if AtsTest.group is None:
AtsTest.groupCounter += 1
self.group = AtsTestGroup(AtsTest.groupCounter)
else:
self.group = AtsTest.group
self.group.append(self)
self.groupNumber = self.group.number
self.groupSerialNumber = len(self.group)
self.waitUntil = AtsTest.waitUntil #never modify this, it may be shared.
self.runOrder = 0 # to aid in diagnosis of wait, priority
self.depends_on = None
self.dependents = []
self.expectedResult = PASSED
self.setName("uninitialized")
self.set(INVALID, "New test, unitialized")
self.srunRelativeNode = -1
self.numNodesToUse = -1
self.priority = -1
self.totalPriority = -1
self.startDateTime = curDateTime()
self.endDateTime = curDateTime()
self.output = [] #magic output, newlines and magic removed.
self.notes = [] #note from the run
self.block = ''
# these will all get changed below but want them set to something for getResults
self.level = 0
self.independent = False
self.np = 1
self.priority = 1
self.totalPriority = 1
self.directory = ''
self.batch = False
self.clas = ''
self.combineOutput = False
self.outname = ''
self.shortoutname = ''
self.errname = ''
self.outhandle = None
self.errhandle = None
self.commandList = ['not run']
# this is just used for documentation
self.commandLine = 'not run'
rootdict = dict(ATSROOT=configuration.ATSROOT)
# Combine the options: first the defaults, then the glued, then the tacked,
# then the stuck, then the test options.
self.options = AttributeDict(
script='',
clas=[],
executable='',
directory='',
)
try:
self.options.update(configuration.options.testDefaults)
self.options.update(AtsTest.glued)
self.options.update(AtsTest.tacked)
self.options.update(AtsTest.stuck)
self.options.update(AtsTest.grouped)
self.options.update(options)
except Exception as e:
self.set(INVALID, 'Bad options: ' + e)
return
self.level = self.options['level']
self.np = self.options['np']
self.priority = self.options.get('priority', max(1, self.np))
self.totalPriority = self.priority
self.testStdout = self.options['testStdout']
outOpts = ['file', 'terminal', 'both']
if not self.testStdout in outOpts:
msg = 'Invalid setting for option testStdout: ' + self.testStdout
raise AtsError(msg)
if configuration.options.allInteractive:
self.batch = False
else:
self.batch = self.options['batch']
if configuration.options.combineOutErr:
self.combineOutput = True
else:
self.combineOutput = False
# process the arguments
# Note: old interface was script, clas='', **options
# Now allow for possibility of no script, or clas as unnamed second
# positional
lc = len(fixedargs)
if lc > 2:
self.set(INVALID, 'Too many positional arguments to test command.')
return
elif lc == 2:
self.options['script'] = fixedargs[0]
self.options['clas'] = fixedargs[1]
elif lc == 1:
self.options['script'] = fixedargs[0]
script = self.options['script']
clas = self.options['clas']
if isinstance(clas, str):
clas = configuration.machine.split(clas)
self.clas = [c % self.options for c in clas]
executable = str(self.options.get('executable'))
self.directory = self.options['directory']
if executable == '1':
if not script:
self.set(INVALID, "executable = 1 requires a first argument.")
return
script = script.replace('$ATSROOT', configuration.ATSROOT)
if len(configuration.ATSROOT) == 0:
script = script[1:] # remove leading "/" or "\"
script = script % rootdict
self.executable = Executable(script)
if self.directory == '':
self.directory = os.getcwd()
path = self.executable.path
junk, filename = os.path.split(path)
else:
if executable:
executable = executable.replace('$ATSROOT',
configuration.ATSROOT)
self.executable = Executable(executable % rootdict)
else:
self.executable = configuration.defaultExecutable
if script:
script = abspath(script) % self.options
self.clas.insert(0, script)
if self.directory == '':
self.directory, filename = os.path.split(script)
else:
if self.directory == '':
self.directory = os.getcwd()
junk, filename = os.path.split(self.executable.path)
name, junk = os.path.splitext(filename)
self.setName(self.options.get('name', name))
label = self.options.get('label', '')
if label:
label = str(label).strip()
self.setName(self.name + '(' + label + ')')
if debug():
log("Results of parsing test arguments", echo=False)
log.indent()
log("Name:", self.name, echo=False)
log("Options:", echo=False)
log.indent()
for k in self.options:
log(k, ": ", self.options[k], echo=False)
log.dedent()
log("Executable path:", self.executable.path, echo=False)
log("Directory:", self.directory, echo=False)
log.dedent()
self.independent = self.options.get('independent', False)
if not self.independent:
# the lower() is due to peculiarities on at least the Mac
# where os.chdir() seems to change case partially.
self.block = self.directory.lower()
if not self.executable.is_valid():
self.set(INVALID, 'Executable "%s" not valid.' % self.executable)
return
if not os.path.isdir(self.directory):
self.set(INVALID, 'Directory not valid: %s' % self.directory)
if script and not is_valid_file(script):
self.set(INVALID, "Script %s does not exist." % script)
return
self.fileOutNamesSet()
#set the timelimit
try:
tl = options.get('timelimit', None)
if tl is None:
self.timelimit = configuration.timelimit
else:
self.timelimit = Duration(tl)
except AtsError as msg:
self.set(INVALID, msg)
return
if self.priority <= 0:
self.set(SKIPPED, 'Test has priority <= zero.')
return
# if the test ends up BATCHED, such jobs are legal.
if self.batch and configuration.options.nobatch:
self.set(SKIPPED, "Batch not available")
elif self.batch:
problem = configuration.batchmachine.canRun(self)
if not problem:
if configuration.options.skip:
self.set(SKIPPED, "BACH skipped due to skip flag")
else:
self.set(BATCHED, "Ready to run in batch.")
else:
self.set(SKIPPED, problem)
else:
problem = configuration.machine.canRun(self)
if not problem:
self.set(CREATED, "Ready to run interactively.")
elif configuration.options.allInteractive or \
configuration.options.nobatch or \
self.groupNumber:
self.set(SKIPPED, problem)
else:
self.set(BATCHED, problem)
self.notes.append(\
"Changed to batch since unable to run interactively on this machine.")
def _source(self, path, introspector, vocabulary):
"Process source file. Returns true if successful"
here = os.getcwd()
t = abspath(path)
directory, filename = os.path.split(t)
name, e = os.path.splitext(filename)
if e:
namelist = [t]
else:
namelist = [t, t + '.ats', t + '.py']
for t1 in namelist:
if t1 in AtsManager.alreadysourced:
log("Already sourced:", t1)
return
try:
f = open(t1)
break
except IOError as e:
pass
else:
log("Error opening input file:", t1, echo=True)
self.badlist.append(t1)
raise AtsError("Could not open input file %s" % path)
t = abspath(t1)
directory, filename = os.path.split(t1)
name, e = os.path.splitext(filename)
AtsManager.alreadysourced.append(t1)
# save to restore after this file is read
savestuck = dict(AtsTest.stuck)
savetacked = dict(AtsTest.tacked)
unstick() #clear sticky list at the start of a file.
AtsTest.waitNewSource()
testenv = dict(testEnvironment)
testenv.update(vocabulary)
testenv['SELF'] = t1
atstext = []
for line1 in f:
if not line1: continue
if line1.startswith('#!'):
continue
magic = introspector(line1[:-1])
if magic is not None:
atstext.append(magic)
f.close()
if atstext:
log('-> Executing statements in', t1, echo=False)
log.indent()
code = '\n'.join(atstext)
if debug():
for line in atstext:
log(line, echo=False)
os.chdir(directory)
try:
exec(code, testenv)
if debug():
log('Finished ', t1, datestamp())
except KeyboardInterrupt:
raise
except Exception as details:
self.badlist.append(t1)
log('Error while processing statements in', t1, ':', echo=True)
log(details, echo=True)
log.dedent()
else:
log('-> Sourcing', t1, echo=False)
log.indent()
os.chdir(directory)
try:
exec(compile(open(t1, "rb").read(), t1, 'exec'), testenv)
if debug(): log('Finished ', t1, datestamp())
result = 1
except KeyboardInterrupt:
raise
except Exception as details:
self.badlist.append(t1)
log('Error in input file', t1, ':', echo=True)
log(details, echo=True)
log('------------------------------------------', echo=True)
log.dedent()
AtsTest.endGroup()
unstick()
stick(**savestuck)
untack()
tack(**savetacked)
AtsTest.waitEndSource()
os.chdir(here)