def parse_test_status(file_contents):
###############################################################################
"""
Returns {ordered dict of phase->status}, test_name
"""
rv = OrderedDict()
test_name = None
for line in file_contents.splitlines():
if (line.split()[0] == COMMENT_STATUS):
pass # skip comments
elif (len(line.split()) == 3):
status, curr_test_name, phase = line.split()
if (test_name is None):
test_name = curr_test_name
if (phase in rv):
# Phase names don't matter here, just need something unique
rv[phase] = reduce_stati({
"%s_" % phase: status,
phase: rv[phase]
})
else:
rv[phase] = status
else:
warning(
"In TestStatus file for test '%s', line '%s' not in expected format"
% (test_name, line))
return rv, test_name
def _setup_cs_files(self):
###########################################################################
try:
acme_scripts_root = acme_util.get_acme_scripts_root()
template_file = os.path.join(acme_scripts_root, "cs.status.template")
template = open(template_file, "r").read()
template = template.replace("<PATH>", acme_scripts_root).replace("<TESTID>", self._test_id)
cs_status_file = os.path.join(self._test_root, "cs.status.%s" % self._test_id)
with open(cs_status_file, "w") as fd:
fd.write(template)
os.chmod(cs_status_file, os.stat(cs_status_file).st_mode | stat.S_IXUSR | stat.S_IXGRP)
template_file = os.path.join(acme_scripts_root, "cs.submit.template")
template = open(template_file, "r").read()
build_cmd = "./*.test_build" if self._no_build else ":"
run_cmd = "./*.test" if self._no_batch else "./*.submit"
template = template.replace("<BUILD_CMD>", build_cmd).replace("<RUN_CMD>", run_cmd).replace("<TESTID>", self._test_id)
if (self._no_build or self._no_run):
cs_submit_file = os.path.join(self._test_root, "cs.submit.%s" % self._test_id)
with open(cs_submit_file, "w") as fd:
fd.write(template)
os.chmod(cs_submit_file, os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP)
except Exception as e:
warning("FAILED to set up cs files: %s" % str(e))
def interpret_status(file_contents, check_throughput=False, check_memory=False, ignore_namelists=False):
###############################################################################
r"""
>>> interpret_status('PASS testname RUN')
('testname', 'PASS')
>>> interpret_status('PASS testname BUILD\nPEND testname RUN')
('testname', 'PEND')
>>> interpret_status('FAIL testname BUILD\nPEND testname RUN')
('testname', 'PEND')
>>> interpret_status('PASS testname BUILD\nPASS testname RUN')
('testname', 'PASS')
>>> interpret_status('PASS testname RUN\nFAIL testname tputcomp')
('testname', 'PASS')
>>> interpret_status('PASS testname RUN\nFAIL testname tputcomp', check_throughput=True)
('testname', 'FAIL')
>>> interpret_status('PASS testname RUN\nNLFAIL testname nlcomp')
('testname', 'NLFAIL')
>>> interpret_status('PASS testname RUN\nNLFAIL testname nlcomp', ignore_namelists=True)
('testname', 'PASS')
>>> interpret_status('PASS testname compare\nNLFAIL testname nlcomp\nFAIL testname compare')
('testname', 'DIFF')
"""
statuses, test_name = parse_test_status(file_contents)
reduced_status = reduce_stati(statuses, check_throughput, check_memory, ignore_namelists)
if (RUN_PHASE not in statuses.keys() and reduced_status != TEST_FAIL_STATUS):
warning("Very odd: Waiting for test '%s' that has no run phase but did not fail?!?!" % test_name)
return test_name, reduced_status
def _handle_test_status_file(self, test_name, test_phase, success):
###########################################################################
#
# This complexity is due to sharing of TestStatus responsibilities
#
try:
if (test_phase != RUN_PHASE and
(not success or test_phase == BUILD_PHASE or test_phase == self._phases[-1])):
self._update_test_status_file(test_name)
# If we failed VERY early on in the run phase, it's possible that
# the CIME scripts never got a chance to set the state.
elif (test_phase == RUN_PHASE and not success):
test_status_file = os.path.join(self._get_test_dir(test_name), TEST_STATUS_FILENAME)
statuses = wait_for_tests.parse_test_status_file(test_status_file)[0]
if ( RUN_PHASE not in statuses or
statuses[RUN_PHASE] in [TEST_PASS_STATUS, TEST_PENDING_STATUS] ):
self._update_test_status_file(test_name)
except Exception as e:
# TODO: What to do here? This failure is very severe because the
# only way for test results to be communicated is by the TestStatus
# file.
warning("VERY BAD! Could not handle TestStatus file '%s': '%s'" % (test_status_file, str(e)))
thread.interrupt_main()
def wait_for_tests_impl(test_paths, no_wait=False, check_throughput=False, check_memory=False, ignore_namelists=False):
###############################################################################
results = Queue.Queue()
for test_path in test_paths:
t = threading.Thread(target=wait_for_test, args=(test_path, results, not no_wait, check_throughput, check_memory, ignore_namelists))
t.daemon = True
t.start()
while threading.active_count() > 1:
time.sleep(1)
test_results = {}
completed_test_paths = []
while (not results.empty()):
test_name, test_path, test_status = results.get()
if (test_name in test_results):
prior_path, prior_status = test_results[test_name]
if (test_status == prior_status):
warning("Test name '%s' was found in both '%s' and '%s'" %
(test_name, test_path, prior_path))
else:
raise SystemExit("Test name '%s' was found in both '%s' and '%s' with different results" %
(test_name, test_path, prior_path))
test_results[test_name] = (test_path, test_status)
completed_test_paths.append(test_path)
expect(set(test_paths) == set(completed_test_paths),
"Missing results for test paths: %s" % (set(test_paths) - set(completed_test_paths)) )
return test_results
def _handle_test_status_file(self, test_name, test_phase, success):
###########################################################################
#
# This complexity is due to sharing of TestStatus responsibilities
#
try:
if (test_phase != RUN_PHASE
and (not success or test_phase == BUILD_PHASE
or test_phase == self._phases[-1])):
self._update_test_status_file(test_name)
# If we failed VERY early on in the run phase, it's possible that
# the CIME scripts never got a chance to set the state.
elif (test_phase == RUN_PHASE and not success):
test_status_file = os.path.join(self._get_test_dir(test_name),
TEST_STATUS_FILENAME)
statuses = wait_for_tests.parse_test_status_file(
test_status_file)[0]
if (RUN_PHASE not in statuses or statuses[RUN_PHASE]
in [TEST_PASS_STATUS, TEST_PENDING_STATUS]):
self._update_test_status_file(test_name)
except Exception as e:
# TODO: What to do here? This failure is very severe because the
# only way for test results to be communicated is by the TestStatus
# file.
warning("VERY BAD! Could not handle TestStatus file '%s': '%s'" %
(os.path.join(self._get_test_dir(test_name),
TEST_STATUS_FILENAME), str(e)))
thread.interrupt_main()
def _setup_cs_files(self):
###########################################################################
try:
acme_scripts_root = acme_util.get_acme_scripts_root()
template_file = os.path.join(acme_scripts_root, "cs.status.template")
template = open(template_file, "r").read()
template = template.replace("<PATH>", acme_scripts_root).replace("<TESTID>", self._test_id)
cs_status_file = os.path.join(self._test_root, "cs.status.%s" % self._test_id)
with open(cs_status_file, "w") as fd:
fd.write(template)
os.chmod(cs_status_file, os.stat(cs_status_file).st_mode | stat.S_IXUSR | stat.S_IXGRP)
template_file = os.path.join(acme_scripts_root, "cs.submit.template")
template = open(template_file, "r").read()
build_cmd = "./*.test_build" if self._no_build else ":"
run_cmd = "./*.test" if self._no_batch else "./*.submit"
template = template.replace("<BUILD_CMD>", build_cmd).replace("<RUN_CMD>", run_cmd).replace("<TESTID>", self._test_id)
if (self._no_build or self._no_run):
cs_submit_file = os.path.join(self._test_root, "cs.submit.%s" % self._test_id)
with open(cs_submit_file, "w") as fd:
fd.write(template)
os.chmod(cs_submit_file, os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP)
except Exception as e:
warning("FAILED to set up cs files: %s" % str(e))
def get_test_output(test_path):
###############################################################################
output_file = os.path.join(test_path, "TestStatus.log")
if (os.path.exists(output_file)):
return open(output_file, 'r').read()
else:
warning("File '%s' not found" % output_file)
return ""
def get_test_output(test_path):
###############################################################################
output_file = os.path.join(test_path, "TestStatus.log")
if (os.path.exists(output_file)):
return open(output_file, 'r').read()
else:
warning("File '%s' not found" % output_file)
return ""
def get_test_time(test_path):
###############################################################################
cmd = "grep TIME %s" % os.path.join(test_path, TEST_STATUS_FILENAME)
stat, output, _ = acme_util.run_cmd(cmd, ok_to_fail=True)
if (stat == 0):
return int(output.split()[-1])
else:
warning("No timing data found in %s" % test_path)
return 0
def get_test_time(test_path):
###############################################################################
cmd = "grep TIME %s" % os.path.join(test_path, TEST_STATUS_FILENAME)
stat, output, _ = acme_util.run_cmd(cmd, ok_to_fail=True)
if (stat == 0):
return int(output.split()[-1])
else:
warning("No timing data found in %s" % test_path)
return 0
def _run_catch_exceptions(self, test_name, phase, run):
###########################################################################
try:
return run(test_name)
except Exception as e:
exc_tb = sys.exc_info()[2]
errput = "Test '%s' failed in phase '%s' with exception '%s'" % (test_name, phase, str(e))
self._log_output(test_name, errput)
warning("Caught exception: %s" % str(e))
traceback.print_tb(exc_tb)
return False
def _run_catch_exceptions(self, test_name, phase, run):
###########################################################################
try:
return run(test_name)
except Exception as e:
exc_tb = sys.exc_info()[2]
errput = "Test '%s' failed in phase '%s' with exception '%s'" % (test_name, phase, str(e))
self._log_output(test_name, errput)
warning("Caught exception: %s" % str(e))
traceback.print_tb(exc_tb)
return False
def wait_for_tests_impl(test_paths,
no_wait=False,
check_throughput=False,
check_memory=False,
ignore_namelists=False):
###############################################################################
results = Queue.Queue()
for test_path in test_paths:
t = threading.Thread(target=wait_for_test,
args=(test_path, results, not no_wait,
check_throughput, check_memory,
ignore_namelists))
t.daemon = True
t.start()
while threading.active_count() > 1:
time.sleep(1)
test_results = {}
completed_test_paths = []
while (not results.empty()):
test_name, test_path, test_status = results.get()
if (test_name in test_results):
prior_path, prior_status = test_results[test_name]
if (test_status == prior_status):
warning("Test name '%s' was found in both '%s' and '%s'" %
(test_name, test_path, prior_path))
else:
raise SystemExit(
"Test name '%s' was found in both '%s' and '%s' with different results"
% (test_name, test_path, prior_path))
test_results[test_name] = (test_path, test_status)
completed_test_paths.append(test_path)
expect(
set(test_paths) == set(completed_test_paths),
"Missing results for test paths: %s" %
(set(test_paths) - set(completed_test_paths)))
return test_results
def parse_test_status(file_contents):
###############################################################################
"""
Returns {ordered dict of phase->status}, test_name
"""
rv = OrderedDict()
test_name = None
for line in file_contents.splitlines():
if (line.split()[0] == COMMENT_STATUS):
pass # skip comments
elif (len(line.split()) == 3):
status, curr_test_name, phase = line.split()
if (test_name is None):
test_name = curr_test_name
if (phase in rv):
# Phase names don't matter here, just need something unique
rv[phase] = reduce_stati({"%s_" % phase : status, phase : rv[phase]})
else:
rv[phase] = status
else:
warning("In TestStatus file for test '%s', line '%s' not in expected format" % (test_name, line))
return rv, test_name
def interpret_status(file_contents,
check_throughput=False,
check_memory=False,
ignore_namelists=False):
###############################################################################
r"""
>>> interpret_status('PASS testname RUN')
('testname', 'PASS')
>>> interpret_status('PASS testname BUILD\nPEND testname RUN')
('testname', 'PEND')
>>> interpret_status('FAIL testname BUILD\nPEND testname RUN')
('testname', 'PEND')
>>> interpret_status('PASS testname BUILD\nPASS testname RUN')
('testname', 'PASS')
>>> interpret_status('PASS testname RUN\nFAIL testname tputcomp')
('testname', 'PASS')
>>> interpret_status('PASS testname RUN\nFAIL testname tputcomp', check_throughput=True)
('testname', 'FAIL')
>>> interpret_status('PASS testname RUN\nNLFAIL testname nlcomp')
('testname', 'NLFAIL')
>>> interpret_status('PASS testname RUN\nNLFAIL testname nlcomp', ignore_namelists=True)
('testname', 'PASS')
>>> interpret_status('PASS testname compare\nNLFAIL testname nlcomp\nFAIL testname compare')
('testname', 'DIFF')
"""
statuses, test_name = parse_test_status(file_contents)
reduced_status = reduce_stati(statuses, check_throughput, check_memory,
ignore_namelists)
if (RUN_PHASE not in statuses.keys()
and reduced_status != TEST_FAIL_STATUS):
warning(
"Very odd: Waiting for test '%s' that has no run phase but did not fail?!?!"
% test_name)
return test_name, reduced_status
def create_cdash_xml(results, cdash_build_name, cdash_project,
cdash_build_group, start_time):
###############################################################################
#
# Create dart config file
#
if (start_time is None):
warning("No valid start_time provided, using current time instead")
start_time = time.time()
utc_time_tuple = time.gmtime(start_time)
cdash_timestamp = time.strftime("%H:%M:%S", utc_time_tuple)
hostname = acme_util.probe_machine_name()
if (hostname is None):
hostname = socket.gethostname().split(".")[0]
warning("Could not convert hostname '%s' into an ACME machine name" %
(hostname))
dart_config = \
"""
SourceDirectory: %s
BuildDirectory: %s
# Site is something like machine.domain, i.e. pragmatic.crd
Site: %s
# Build name is osname-revision-compiler, i.e. Linux-2.4.2-2smp-c++
BuildName: %s
# Submission information
IsCDash: TRUE
CDashVersion:
QueryCDashVersion:
DropSite: my.cdash.org
DropLocation: /submit.php?project=%s
DropSiteUser:
DropSitePassword:
DropSiteMode:
DropMethod: http
TriggerSite:
ScpCommand: %s
# Dashboard start time
NightlyStartTime: %s UTC
""" % (os.getcwd(), os.getcwd(), hostname,
cdash_build_name, cdash_project, distutils.spawn.find_executable("scp"), cdash_timestamp)
with open("DartConfiguration.tcl", "w") as dart_fd:
dart_fd.write(dart_config)
utc_time = time.strftime('%Y%m%d-%H%M', utc_time_tuple)
os.makedirs(os.path.join("Testing", utc_time))
# Make tag file
with open("Testing/TAG", "w") as tag_fd:
tag_fd.write("%s\n%s\n" % (utc_time, cdash_build_group))
create_cdash_test_xml(results, cdash_build_name, cdash_build_group,
utc_time, start_time, hostname)
create_cdash_upload_xml(results, cdash_build_name, cdash_build_group,
utc_time, hostname)
acme_util.run_cmd("ctest -VV -D NightlySubmit", verbose=True)
def create_cdash_xml(results, cdash_build_name, cdash_project, cdash_build_group, start_time):
###############################################################################
#
# Create dart config file
#
if (start_time is None):
warning("No valid start_time provided, using current time instead")
start_time = time.time()
utc_time_tuple = time.gmtime(start_time)
cdash_timestamp = time.strftime("%H:%M:%S", utc_time_tuple)
hostname = acme_util.probe_machine_name()
if (hostname is None):
hostname = socket.gethostname().split(".")[0]
warning("Could not convert hostname '%s' into an ACME machine name" % (hostname))
dart_config = \
"""
SourceDirectory: %s
BuildDirectory: %s
# Site is something like machine.domain, i.e. pragmatic.crd
Site: %s
# Build name is osname-revision-compiler, i.e. Linux-2.4.2-2smp-c++
BuildName: %s
# Submission information
IsCDash: TRUE
CDashVersion:
QueryCDashVersion:
DropSite: my.cdash.org
DropLocation: /submit.php?project=%s
DropSiteUser:
DropSitePassword:
DropSiteMode:
DropMethod: http
TriggerSite:
ScpCommand: %s
# Dashboard start time
NightlyStartTime: %s UTC
""" % (os.getcwd(), os.getcwd(), hostname,
cdash_build_name, cdash_project, distutils.spawn.find_executable("scp"), cdash_timestamp)
with open("DartConfiguration.tcl", "w") as dart_fd:
dart_fd.write(dart_config)
utc_time = time.strftime('%Y%m%d-%H%M', utc_time_tuple)
os.makedirs(os.path.join("Testing", utc_time))
# Make tag file
with open("Testing/TAG", "w") as tag_fd:
tag_fd.write("%s\n%s\n" % (utc_time, cdash_build_group))
create_cdash_test_xml(results, cdash_build_name, cdash_build_group, utc_time, start_time, hostname)
create_cdash_upload_xml(results, cdash_build_name, cdash_build_group, utc_time, hostname)
acme_util.run_cmd("ctest -VV -D NightlySubmit", verbose=True)