def handleJobStatus(self, job):
""" Method to handle a job status """
tester = job.getTester()
if not tester.isSilent():
# Print results and perform any desired post job processing
if job.isFinished():
tester = self.normalizeStatus(job)
# perform printing of application output if so desired
self.printOutput(job)
# Print status with caveats
print(util.formatResult(job, self.options, caveats=True))
timing = job.getTiming()
# Save these results for 'Final Test Result' summary
self.test_table.append( (job, tester.getStatus().status, timing) )
self.postRun(tester.specs, timing)
if tester.isSkip():
self.num_skipped += 1
elif tester.isPass():
self.num_passed += 1
elif tester.isFail():
self.num_failed += 1
else:
self.num_pending += 1
# Just print current status without saving results
else:
print(util.formatResult(job, self.options, result='RUNNING', caveats=False))
def handleJobStatus(self, job):
""" Method to handle a job status """
tester = job.getTester()
if not tester.isSilent():
# Print results and perform any desired post job processing
if job.isFinished():
tester = self.normalizeStatus(job)
# perform printing of application output if so desired
self.printOutput(job)
# Print status with caveats
print(util.formatResult(job, self.options, caveats=True))
timing = job.getTiming()
# Save these results for 'Final Test Result' summary
self.test_table.append( (job, tester.getStatus().status, timing) )
self.postRun(tester.specs, timing)
if tester.isSkip():
self.num_skipped += 1
elif tester.isPass():
self.num_passed += 1
elif tester.isFail():
self.num_failed += 1
else:
self.num_pending += 1
# Just print current status without saving results
else:
print(util.formatResult(job, self.options, result='RUNNING', caveats=False))
def handleTestStatus(self, tester_data):
""" Method to handle a testers status """
tester = tester_data.getTester()
# print and store those results
result = self.printResult(tester_data)
# Test is finished and had some results to print
if result and tester.isFinished():
timing = tester_data.getTiming()
# Store these results to a table we will use when we print final results
self.test_table.append((tester_data, result, timing))
self.postRun(tester.specs, timing)
# Tally the results of this test to be used in our Final Test Results footer
if tester.isSkipped():
self.num_skipped += 1
elif tester.didPass():
self.num_passed += 1
elif tester.isQueued() or tester.isWaiting():
self.num_pending += 1
else:
self.num_failed += 1
# Write results to a file if asked to do so
if not tester.isSkipped():
if not tester.didPass() and not self.options.failed_tests:
self.writeFailedTest.write(tester.getTestName() + '\n')
if self.options.file:
self.file.write(
util.formatResult(
tester_data, result, self.options, color=False) +
'\n')
if self.options.sep_files or (self.options.fail_files
and not tester.didPass()) or (
self.options.ok_files
and tester.didPass()):
fname = os.path.join(
tester.getTestDir(),
tester.getTestName().split('/')[-1] + '.' +
result[:6] + '.txt')
f = open(fname, 'w')
f.write(
util.formatResult(tester_data,
tester_data.getOutput(),
self.options,
color=False) + '\n')
f.close()
def printResult(self, tester_data):
""" Method to print a testers status to the screen """
tester = tester_data.getTester()
# The test has no status to print
if tester.isSilent() or (tester.isDeleted() and not self.options.extra_info):
caveat_formatted_results = None
# Print what ever status the tester has at the time
else:
if self.options.verbose or (tester.didFail() and not self.options.quiet):
output = 'Working Directory: ' + tester.getTestDir() + '\nRunning command: ' + tester.getCommand(self.options) + '\n'
output += tester_data.getOutput()
output = output.replace('\r', '\n') # replace the carriage returns with newlines
lines = output.split('\n')
# Obtain color based on test status
color = tester.getColor()
if output != '':
test_name = util.colorText(tester.getTestName() + ": ", color, colored=self.options.colored, code=self.options.code)
output = test_name + ("\n" + test_name).join(lines)
print(output)
caveat_formatted_results = self.formatCaveats(tester)
print(util.formatResult(tester_data, caveat_formatted_results, self.options))
return caveat_formatted_results
def printResult(self, tester_data):
""" Method to print a testers status to the screen """
tester = tester_data.getTester()
caveat_formatted_results = None
# Print what ever status the tester has at the time
if self.canPrint(tester):
if self.options.verbose or (tester.didFail()
and not self.options.quiet):
output = 'Working Directory: ' + tester.getTestDir(
) + '\nRunning command: ' + tester.getCommand(
self.options) + '\n'
output += tester_data.getOutput()
output = output.replace(
'\r', '\n') # replace the carriage returns with newlines
lines = output.split('\n')
# Obtain color based on test status
color = tester.getColor()
if output != '':
test_name = util.colorText(tester.getTestName() + ": ",
color,
colored=self.options.colored,
code=self.options.code)
output = test_name + ("\n" + test_name).join(lines)
print(output)
caveat_formatted_results = self.formatCaveats(tester)
print(
util.formatResult(tester_data, caveat_formatted_results,
self.options))
return caveat_formatted_results
def cleanup(self):
# Print the results table again if a bunch of output was spewed to the screen between
# tests as they were running
if (self.options.verbose or (self.num_failed != 0 and not self.options.quiet)) and not self.options.dry_run:
print('\n\nFinal Test Results:\n' + ('-' * (util.TERM_COLS-1)))
for (tester_data, result, timing) in sorted(self.test_table, key=lambda x: x[1], reverse=True):
print(util.formatResult(tester_data, result, self.options))
time = clock() - self.start_time
print('-' * (util.TERM_COLS-1))
# Mask off TestHarness error codes to report parser errors
fatal_error = ''
if self.error_code & Parser.getErrorCodeMask():
fatal_error += ', <r>FATAL PARSER ERROR</r>'
if self.error_code & ~Parser.getErrorCodeMask():
fatal_error += ', <r>FATAL TEST HARNESS ERROR</r>'
# Alert the user to their session file
if self.options.queueing:
print 'Your session file is %s' % self.options.session_file
# Print a different footer when performing a dry run
if self.options.dry_run:
print('Processed %d tests in %.1f seconds' % (self.num_passed+self.num_skipped, time))
summary = '<b>%d would run</b>'
summary += ', <b>%d would be skipped</b>'
summary += fatal_error
print(util.colorText( summary % (self.num_passed, self.num_skipped), "", html = True, \
colored=self.options.colored, code=self.options.code ))
else:
print('Ran %d tests in %.1f seconds' % (self.num_passed+self.num_failed, time))
if self.num_passed:
summary = '<g>%d passed</g>'
else:
summary = '<b>%d passed</b>'
summary += ', <b>%d skipped</b>'
if self.num_pending:
summary += ', <c>%d pending</c>'
else:
summary += ', <b>%d pending</b>'
if self.num_failed:
summary += ', <r>%d FAILED</r>'
else:
summary += ', <b>%d failed</b>'
summary += fatal_error
print(util.colorText( summary % (self.num_passed, self.num_skipped, self.num_pending, self.num_failed), "", html = True, \
colored=self.options.colored, code=self.options.code ))
if self.file:
self.file.close()
# Close the failed_tests file
if self.writeFailedTest != None:
self.writeFailedTest.close()
def cleanup(self):
# Print the results table again if a bunch of output was spewed to the screen between
# tests as they were running
if (self.options.verbose or
(self.num_failed != 0
and not self.options.quiet)) and not self.options.dry_run:
print('\n\nFinal Test Results:\n' + ('-' * (util.TERM_COLS - 1)))
for (tester_data, result, timing) in sorted(self.test_table,
key=lambda x: x[1],
reverse=True):
print(util.formatResult(tester_data, result, self.options))
time = clock() - self.start_time
print('-' * (util.TERM_COLS - 1))
# Mask off TestHarness error codes to report parser errors
fatal_error = ''
if self.error_code & Parser.getErrorCodeMask():
fatal_error += ', <r>FATAL PARSER ERROR</r>'
if self.error_code & ~Parser.getErrorCodeMask():
fatal_error += ', <r>FATAL TEST HARNESS ERROR</r>'
# Print a different footer when performing a dry run
if self.options.dry_run:
print('Processed %d tests in %.1f seconds' %
(self.num_passed + self.num_skipped, time))
summary = '<b>%d would run</b>'
summary += ', <b>%d would be skipped</b>'
summary += fatal_error
print(util.colorText( summary % (self.num_passed, self.num_skipped), "", html = True, \
colored=self.options.colored, code=self.options.code ))
else:
print('Ran %d tests in %.1f seconds' %
(self.num_passed + self.num_failed, time))
if self.num_passed:
summary = '<g>%d passed</g>'
else:
summary = '<b>%d passed</b>'
summary += ', <b>%d skipped</b>'
if self.num_failed:
summary += ', <r>%d FAILED</r>'
else:
summary += ', <b>%d failed</b>'
summary += fatal_error
print(util.colorText( summary % (self.num_passed, self.num_skipped, self.num_failed), "", html = True, \
colored=self.options.colored, code=self.options.code ))
if self.file:
self.file.close()
# Close the failed_tests file
if self.writeFailedTest != None:
self.writeFailedTest.close()
def handleTestStatus(self, tester_data):
""" Method to handle a testers status """
tester = tester_data.getTester()
# print and store those results
result = self.printResult(tester_data)
# Test is finished and had some results to print
if result and tester.isFinished():
timing = tester_data.getTiming()
# Store these results to a table we will use when we print final results
self.test_table.append( (tester_data, result, timing) )
self.postRun(tester.specs, timing)
# Tally the results of this test to be used in our Final Test Results footer
if tester.isSkipped():
self.num_skipped += 1
elif tester.didPass():
self.num_passed += 1
elif tester.isQueued() or tester.isWaiting():
self.num_pending += 1
else:
self.num_failed += 1
# Write results to a file if asked to do so
if not tester.isSkipped():
if not tester.didPass() and not self.options.failed_tests:
self.writeFailedTest.write(tester.getTestName() + '\n')
if self.options.file:
self.file.write(util.formatResult( tester_data, result, self.options, color=False) + '\n')
if self.options.sep_files or (self.options.fail_files and not tester.didPass()) or (self.options.ok_files and tester.didPass()):
fname = os.path.join(tester.getTestDir(), tester.getTestName().split('/')[-1] + '.' + result[:6] + '.txt')
f = open(fname, 'w')
f.write(util.formatResult( tester_data, tester_data.getOutput(), self.options, color=False) + '\n')
f.close()
def handleJobStatus(self, job):
"""
The Scheduler is calling back the TestHarness to inform us of a status change.
The job may or may not be finished yet (RUNNING), or failing, passing, etc.
"""
if not job.isSilent():
# Print results and perform any desired post job processing
if job.isFinished():
status, message, color, exit_code = job.getJointStatus()
self.error_code = self.error_code | exit_code
# perform printing of application output if so desired
self.printOutput(job, color)
# Print status with caveats
print(util.formatResult(job, self.options, caveats=True))
timing = job.getTiming()
# Save these results for 'Final Test Result' summary
self.test_table.append( (job, exit_code, timing) )
self.postRun(job.specs, timing)
if job.isSkip():
self.num_skipped += 1
elif job.isPass():
self.num_passed += 1
elif job.isFail():
self.num_failed += 1
else:
self.num_pending += 1
# Just print current status without saving results
else:
print(util.formatResult(job, self.options, result='RUNNING', caveats=False))
def writeResults(self):
""" Don't update the results file when using the --failed-tests argument """
if self.options.failed_tests:
return
""" write test results to disc in some fashion the user has requested """
all_jobs = self.scheduler.retrieveJobs()
# Record the input file name that was used
self.options.results_storage['INPUT_FILE_NAME'] = self.options.input_file_name
# Write some useful data to our results_storage
for job in all_jobs:
tester = job.getTester()
# If queueing, do not store silent results in session file
if tester.isSilent() and self.options.queueing:
continue
# Create empty key based on TestDir, or re-inialize with existing data so we can append to it
self.options.results_storage[tester.getTestDir()] = self.options.results_storage.get(tester.getTestDir(), {})
self.options.results_storage[tester.getTestDir()][tester.getTestName()] = {'NAME' : job.getTestNameShort(),
'LONG_NAME' : tester.getTestName(),
'TIMING' : job.getTiming(),
'STATUS' : tester.getStatus().status,
'FAIL' : tester.isFail(),
'COLOR' : tester.getStatus().color,
'CAVEATS' : list(tester.getCaveats()),
'OUTPUT' : job.getOutput(),
'COMMAND' : tester.getCommand(self.options)}
# Additional data to store (overwrites any previous matching keys)
self.options.results_storage[tester.getTestDir()].update(job.getMetaData())
if self.options.output_dir:
self.results_storage = os.path.join(self.options.output_dir, self.results_storage)
if self.options.results_storage:
try:
with open(self.results_storage, 'w') as data_file:
json.dump(self.options.results_storage, data_file, indent=2)
except UnicodeDecodeError:
print('\nERROR: Unable to write results due to unicode decode/encode error')
# write to a plain file to aid in reproducing error
with open(self.results_storage + '.unicode_error' , 'w') as f:
f.write(self.options.results_storage)
sys.exit(1)
except IOError:
print('\nERROR: Unable to write results due to permissions')
sys.exit(1)
try:
# Write one file, with verbose information (--file)
if self.options.file:
with open(os.path.join(self.output_dir, self.options.file), 'w') as f:
for job in all_jobs:
tester = job.getTester()
# Do not write information about silent tests
if tester.isSilent():
continue
formated_results = util.formatResult( job, self.options, result=job.getOutput(), color=False)
f.write(formated_results + '\n')
# Write a separate file for each test with verbose information (--sep-files, --sep-files-ok, --sep-files-fail)
if ((self.options.ok_files and self.num_passed)
or (self.options.fail_files and self.num_failed)):
for job in all_jobs:
tester = job.getTester()
if self.options.output_dir:
output_dir = self.options.output_dir
else:
output_dir = tester.getTestDir()
# Yes, by design test dir will be apart of the output file name
output_file = os.path.join(output_dir, '.'.join([os.path.basename(tester.getTestDir()),
job.getTestNameShort(),
tester.getStatus().status,
'txt']))
formated_results = util.formatResult(job, self.options, result=job.getOutput(), color=False)
# Passing tests
if self.options.ok_files and tester.isPass():
with open(output_file, 'w') as f:
f.write(formated_results)
# Failing tests
if self.options.fail_files and tester.isFail():
with open(output_file, 'w') as f:
f.write(formated_results)
except IOError:
print('Permission error while writing results to disc')
sys.exit(1)
except:
print('Error while writing results to disc')
sys.exit(1)
def cleanup(self):
# Not interesting in printing any final results if we are cleaning up old queue manager runs
if self.options.queue_cleanup:
try:
os.remove(self.results_storage)
except OSError:
pass
return
# Print the results table again if a bunch of output was spewed to the screen between
# tests as they were running
if len(self.parse_errors) > 0:
print('\n\nParser Errors:\n' + ('-' * (util.TERM_COLS)))
for err in self.parse_errors:
print(util.colorText(err, 'RED', html=True, colored=self.options.colored, code=self.options.code))
if (self.options.verbose or (self.num_failed != 0 and not self.options.quiet)) and not self.options.dry_run:
print('\n\nFinal Test Results:\n' + ('-' * (util.TERM_COLS)))
for (job, result, timing) in sorted(self.test_table, key=lambda x: x[1], reverse=True):
print(util.formatResult(job, self.options, caveats=True))
time = clock() - self.start_time
print('-' * (util.TERM_COLS))
# Mask off TestHarness error codes to report parser errors
fatal_error = ''
if self.error_code:
fatal_error += ', <r>FATAL TEST HARNESS ERROR</r>'
if len(self.parse_errors) > 0:
fatal_error += ', <r>FATAL PARSER ERROR</r>'
self.error_code = 1
# Alert the user to their session file
if self.options.queueing:
print('Your session file is %s' % self.results_storage)
# Print a different footer when performing a dry run
if self.options.dry_run:
print('Processed %d tests in %.1f seconds.' % (self.num_passed+self.num_skipped, time))
summary = '<b>%d would run</b>'
summary += ', <b>%d would be skipped</b>'
summary += fatal_error
print(util.colorText( summary % (self.num_passed, self.num_skipped), "", html = True, \
colored=self.options.colored, code=self.options.code ))
else:
print('Ran %d tests in %.1f seconds.' % (self.num_passed+self.num_failed, time))
if self.num_passed:
summary = '<g>%d passed</g>'
else:
summary = '<b>%d passed</b>'
summary += ', <b>%d skipped</b>'
if self.num_pending:
summary += ', <c>%d pending</c>'
else:
summary += ', <b>%d pending</b>'
if self.num_failed:
summary += ', <r>%d FAILED</r>'
else:
summary += ', <b>%d failed</b>'
if self.scheduler.maxFailures():
summary += '\n<r>MAX FAILURES REACHED</r>'
summary += fatal_error
print(util.colorText( summary % (self.num_passed, self.num_skipped, self.num_pending, self.num_failed), "", html = True, \
colored=self.options.colored, code=self.options.code ))
# Perform any write-to-disc operations
self.writeResults()
def writeResults(self):
""" Don't update the results file when using the --failed-tests argument """
if self.options.failed_tests:
return
""" write test results to disc in some fashion the user has requested """
all_jobs = self.scheduler.retrieveJobs()
# Record the input file name that was used
self.options.results_storage['INPUT_FILE_NAME'] = self.options.input_file_name
# Write some useful data to our results_storage
for job in all_jobs:
tester = job.getTester()
# If queueing, do not store silent results in session file
if tester.isSilent() and self.options.queueing:
continue
# Create empty key based on TestDir, or re-inialize with existing data so we can append to it
self.options.results_storage[tester.getTestDir()] = self.options.results_storage.get(tester.getTestDir(), {})
self.options.results_storage[tester.getTestDir()][tester.getTestName()] = {'NAME' : job.getTestNameShort(),
'LONG_NAME' : tester.getTestName(),
'TIMING' : job.getTiming(),
'STATUS' : tester.getStatus().status,
'FAIL' : tester.isFail(),
'COLOR' : tester.getStatus().color,
'CAVEATS' : list(tester.getCaveats()),
'OUTPUT' : job.getOutput(),
'COMMAND' : tester.getCommand(self.options)}
# Additional data to store (overwrites any previous matching keys)
self.options.results_storage[tester.getTestDir()].update(job.getMetaData())
if self.options.output_dir:
self.results_storage = os.path.join(self.options.output_dir, self.results_storage)
if self.options.results_storage:
try:
with open(self.results_storage, 'w') as data_file:
json.dump(self.options.results_storage, data_file, indent=2)
except UnicodeDecodeError:
print('\nERROR: Unable to write results due to unicode decode/encode error')
# write to a plain file to aid in reproducing error
with open(self.results_storage + '.unicode_error' , 'w') as f:
f.write(self.options.results_storage)
sys.exit(1)
except IOError:
print('\nERROR: Unable to write results due to permissions')
sys.exit(1)
try:
# Write one file, with verbose information (--file)
if self.options.file:
with open(os.path.join(self.output_dir, self.options.file), 'w') as f:
for job in all_jobs:
tester = job.getTester()
# Do not write information about silent tests
if tester.isSilent():
continue
formated_results = util.formatResult( job, self.options, result=job.getOutput(), color=False)
f.write(formated_results + '\n')
# Write a separate file for each test with verbose information (--sep-files, --sep-files-ok, --sep-files-fail)
if ((self.options.ok_files and self.num_passed)
or (self.options.fail_files and self.num_failed)):
for job in all_jobs:
tester = job.getTester()
if self.options.output_dir:
output_dir = self.options.output_dir
else:
output_dir = tester.getTestDir()
# Yes, by design test dir will be apart of the output file name
output_file = os.path.join(output_dir, '.'.join([os.path.basename(tester.getTestDir()),
job.getTestNameShort(),
tester.getStatus().status,
'txt']))
formated_results = util.formatResult(job, self.options, result=job.getOutput(), color=False)
# Passing tests
if self.options.ok_files and tester.isPass():
with open(output_file, 'w') as f:
f.write(formated_results)
# Failing tests
if self.options.fail_files and tester.isFail():
with open(output_file, 'w') as f:
f.write(formated_results)
except IOError:
print('Permission error while writing results to disc')
sys.exit(1)
except:
print('Error while writing results to disc')
sys.exit(1)
def cleanup(self):
# Not interesting in printing any final results if we are cleaning up old queue manager runs
if self.options.queue_cleanup:
try:
os.remove(self.results_storage)
except OSError:
pass
return
# Print the results table again if a bunch of output was spewed to the screen between
# tests as they were running
if len(self.parse_errors) > 0:
print('\n\nParser Errors:\n' + ('-' * (util.TERM_COLS)))
for err in self.parse_errors:
print(util.colorText(err, 'RED', html=True, colored=self.options.colored, code=self.options.code))
if (self.options.verbose or (self.num_failed != 0 and not self.options.quiet)) and not self.options.dry_run:
print('\n\nFinal Test Results:\n' + ('-' * (util.TERM_COLS)))
for (job, result, timing) in sorted(self.test_table, key=lambda x: x[1], reverse=True):
print(util.formatResult(job, self.options, caveats=True))
time = clock() - self.start_time
print('-' * (util.TERM_COLS))
# Mask off TestHarness error codes to report parser errors
fatal_error = ''
if self.error_code:
fatal_error += ', <r>FATAL TEST HARNESS ERROR</r>'
if len(self.parse_errors) > 0:
fatal_error += ', <r>FATAL PARSER ERROR</r>'
self.error_code = 1
# Alert the user to their session file
if self.options.queueing:
print('Your session file is %s' % self.results_storage)
# Print a different footer when performing a dry run
if self.options.dry_run:
print('Processed %d tests in %.1f seconds.' % (self.num_passed+self.num_skipped, time))
summary = '<b>%d would run</b>'
summary += ', <b>%d would be skipped</b>'
summary += fatal_error
print(util.colorText( summary % (self.num_passed, self.num_skipped), "", html = True, \
colored=self.options.colored, code=self.options.code ))
else:
print('Ran %d tests in %.1f seconds.' % (self.num_passed+self.num_failed, time))
if self.num_passed:
summary = '<g>%d passed</g>'
else:
summary = '<b>%d passed</b>'
summary += ', <b>%d skipped</b>'
if self.num_pending:
summary += ', <c>%d pending</c>'
else:
summary += ', <b>%d pending</b>'
if self.num_failed:
summary += ', <r>%d FAILED</r>'
else:
summary += ', <b>%d failed</b>'
if self.scheduler.maxFailures():
summary += '\n<r>MAX FAILURES REACHED</r>'
summary += fatal_error
print(util.colorText( summary % (self.num_passed, self.num_skipped, self.num_pending, self.num_failed), "", html = True, \
colored=self.options.colored, code=self.options.code ))
# Perform any write-to-disc operations
self.writeResults()
