def serialize_and_read(test_suites, to_file=False, prettyprint=None):
"""writes the test suite to an XML string and then re-reads it using minidom,
returning => (test suite element, list of test case elements)"""
try:
iter(test_suites)
except TypeError:
test_suites = [test_suites]
if to_file:
fd, filename = tempfile.mkstemp(text=True)
with os.fdopen(fd, 'w') as f:
TestSuite.to_file(f, test_suites)
print("Serialized XML to temp file [%s]" % filename)
xmldoc = minidom.parse(filename)
os.remove(filename)
else:
if prettyprint is not None:
xml_string = TestSuite.to_xml_string(test_suites, prettyprint=prettyprint)
else:
xml_string = TestSuite.to_xml_string(test_suites)
print("Serialized XML to string:\n%s" % xml_string)
xmldoc = minidom.parseString(xml_string)
ret = []
suites = xmldoc.getElementsByTagName("testsuites")[0]
for suite in suites.getElementsByTagName("testsuite"):
cases = suite.getElementsByTagName("testcase")
ret.append((suite, cases))
return ret
def test_to_xml_string_test_suites_not_a_list(self):
test_suites = TestSuite('suite1', [TestCase('Test1')])
try:
TestSuite.to_xml_string(test_suites)
except Exception as exc:
self.assertEqual(str(exc), 'test_suites must be a list of test suites')
def serialize_and_read(test_suites, to_file=False, prettyprint=None):
"""writes the test suite to an XML string and then re-reads it using minidom,
returning => (test suite element, list of test case elements)"""
try:
iter(test_suites)
except TypeError:
test_suites = [test_suites]
if to_file:
fd, filename = tempfile.mkstemp(text=True)
with os.fdopen(fd, 'w') as f:
TestSuite.to_file(f, test_suites)
print("Serialized XML to temp file [%s]" % filename)
xmldoc = minidom.parse(filename)
os.remove(filename)
else:
if prettyprint is not None:
xml_string = TestSuite.to_xml_string(test_suites,
prettyprint=prettyprint)
else:
xml_string = TestSuite.to_xml_string(test_suites)
print("Serialized XML to string:\n%s" % xml_string)
xmldoc = minidom.parseString(xml_string)
ret = []
suites = xmldoc.getElementsByTagName("testsuites")[0]
for suite in suites.getElementsByTagName("testsuite"):
cases = suite.getElementsByTagName("testcase")
ret.append((suite, cases))
return ret
def test_to_xml_string_test_suites_not_a_list(self):
test_suites = TestSuite('suite1', [TestCase('Test1')])
try:
TestSuite.to_xml_string(test_suites)
except Exception as exc:
self.assertEqual(str(exc),
'test_suites must be a list of test suites')
def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None):
from junit_xml import TestSuite, TestCase
test_suites = []
test_cases = []
for platform in sorted(test_result_ext.keys()):
# {platform : ['Platform', 'Result', 'Scope', 'Description'])
test_cases = []
for tr_result in test_result_ext[platform]:
result, name, scope, description = tr_result
classname = "test.ioper.%s.%s.%s" % (platform, name, scope)
elapsed_sec = 0
_stdout = description
_stderr = ""
# Test case
tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
# Test case extra failure / error info
if result == "FAIL":
tc.add_failure_info(description, _stdout)
elif result == "ERROR":
tc.add_error_info(description, _stdout)
elif result == "SKIP" or result == "NOT_SUPPORTED":
tc.add_skipped_info(description, _stdout)
test_cases.append(tc)
ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases)
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
def main():
parser = argparse.ArgumentParser(description='dummy test')
parser.add_argument('-classes', type=int, default=5, help='number of classes')
parser.add_argument('-testcases', type=int, default=10, help='number of testcases')
parser.add_argument('-pass_rate', type=int, default=75, help='pass rate')
parser.add_argument('-error_rate', type=int, default=20, help='error rate')
parser.add_argument('-failure_rate', type=int, default=10, help='failure rate')
parser.add_argument('-skip_rate', type=int, default=10, help='skip rate')
parser.add_argument('-outputfile', type=str, default='test_results.xml', help='output file')
parser.add_argument('-print', action='store_true', help='print the test results')
args = parser.parse_args()
ts = TestSuite(name='my test suite', hostname=platform.node(), timestamp=datetime.now())
for i in range(args.classes):
for j in range(args.testcases):
tc = TestCase(classname=f"myclass{i}",
name=f"mytest{j}",
elapsed_sec=random.randint(100, 1000),
stdout = "stdout output",
stderr = "stderr output")
if random.randint(0, 100) < args.pass_rate:
if random.randint(0, 100) < args.error_rate:
tc.add_error_info(message=f"error {i} {j}", output="error output message", error_type="ERR1")
elif random.randint(0, 100) < args.failure_rate:
tc.add_failure_info(message=f"failure {i} {j}", output="failure output message", failure_type="FAIL1")
elif random.randint(0, 100) < args.skip_rate:
tc.add_skipped_info(message=f"skipped {i} {j}", output="skipped output message")
ts.test_cases.append(tc)
# pretty printing is on by default but can be disabled using prettyprint=False
if args.print:
print(TestSuite.to_xml_string([ts]))
with open(args.outputfile, 'w') as f:
TestSuite.to_file(f, [ts], prettyprint=True)
def generate_junit_xml(inputfile):
target = None
suite = None
infos = []
errors = []
testcases = []
for line in inputfile:
tag = line[0:3]
props = line[3:].split(':')
if tag == "[!]":
if len(props) == 2:
if props[0].strip().lower() == "target":
target = os.path.basename(props[1].strip())
elif props[0].strip().lower() == "group":
suite = props[1].strip()
else:
infos.append(line)
else:
infos.append(line)
if tag == "[x]":
errors.append(line)
if tag == "[+]":
testcases.append(TestCase(name=props[0].strip(), classname=target, stdout=line))
if tag == "[-]":
tc = TestCase(name=props[0].strip(), classname=target)
tc.add_failure_info(message=props[1].strip(), output=line, failure_type="failed")
testcases.append(tc)
ts = TestSuite(name=suite, test_cases=testcases, stdout="\n".join(infos), stderr="\n".join(errors))
return TestSuite.to_xml_string([ts])
def main():
cwd = os.getcwd()
args = parse_args()
try:
if os.path.exists(args.clairfile):
with open(args.clairfile) as clairfile:
clair_parsed_file = json.load(clairfile)
if os.path.exists(
os.path.join("clair-scanner-logs", "/clair_setup_errors.log")):
with open(
os.path.join("clair-scanner-logs",
"/clair_setup_errors.log"),
'r') as clairfile_errors:
clair_parsed_error_file = clairfile_errors.readlines()
else:
clair_parsed_error_file = None
except:
logger.exception("Failed to parse clair / clair_error file. Exiting.")
current_sorted_level = None
current_suite = None
test_suites = []
if clair_parsed_error_file:
current_suite = TestSuite("SetupError")
new_step = TestCase(name="SetupError",
classname="SetupError",
status="unapproved",
stderr=clair_parsed_error_file)
new_step.log = clair_parsed_error_file
new_step.category = "SetupError"
new_step.failure_type = "unapproved"
new_step.failure_message = "Please have the following security issue reviewed by Splunk: {}".format(
vuln["link"])
new_step.failure_output = clair_parsed_error_file
current_suite.test_cases.append(new_step)
test_suites.append(current_suite)
for vuln in clair_parsed_file["vulnerabilities"]:
if current_sorted_level != vuln["severity"]:
if current_suite:
test_suites.append(current_suite)
current_suite = TestSuite(name=vuln["severity"])
current_sorted_level = vuln["severity"]
new_step = TestCase(name=vuln["vulnerability"],
classname=vuln["severity"],
status="unapproved",
url=vuln["link"],
stderr=vuln["description"])
new_step.log = vuln
new_step.category = vuln["severity"]
new_step.failure_type = "unapproved"
new_step.failure_message = "Please have the following security issue reviewed by Splunk: {}".format(
vuln["link"])
new_step.failure_output = vuln["description"]
current_suite.test_cases.append(new_step)
# try to write new file
try:
with open(args.output, 'w') as outfile:
outfile.write(TestSuite.to_xml_string(test_suites))
except:
logger.exception("Filed saving file.")
def main():
SignalHandler.init()
args = parseArguments()
scriptFiles = getScriptFiles(args.start, args.scriptName)
testSuits = createTestSuits(scriptFiles, args.start)
markTestSkipedByPattern(testSuits, args.pattern)
genTestExecuter = [TestExecuter(ts, tc, args.upgradeScriptFile) for ts in testSuits for tc in ts.test_cases if not tc.is_skipped()]
if (args.jobs == None):
runner = Serial(genTestExecuter, failfast=args.failfast)
else:
runner = Parallel(genTestExecuter, failfast=args.failfast, max_workers = args.jobs)
startTime = time.time()
processStartTime = time.process_time()
res = runner.run(".", printer = Printer(args.verbose != None))
processTime = time.process_time() - processStartTime
executionTime = time.time() - startTime
if (args.failfast and res == False) or SignalHandler.failFast():
for ts in testSuits:
for tc in ts.test_cases:
if tc.elapsed_sec == None:
if (SignalHandler.failFast()):
tc.add_skipped_info("skipped due to " + SignalHandler.signalName())
else:
tc.add_skipped_info("skipped due to --failfast argument")
report = getReport(testSuits, processTime, executionTime)
Printer(args.verbose != None).print(report["text"])
if (args.reportfile):
ensure_dir(args.reportfile)
with open(args.reportfile, "w") as fw:
fw.write(TestSuite.to_xml_string(testSuits))
exit(report["error"] > 0)
def parse(infile, outfile, format_type, classname, suitename):
testcases = list()
testcase_logs = list()
current = None
test_block_delimiter = known_formats[format_type]['tb_delimiter']
# separate log file into test blocks by test block delimiter
for line in infile:
if test_block_delimiter(line):
if current: # non-empty list
testcase_logs.append(current)
current = list()
if current is not None:
current.append(line)
# add last record if present
if current not in testcase_logs:
testcase_logs.append(current)
# create test cases from test blocks
for entry in testcase_logs:
testcases.append(known_formats[format_type]['test_parser'](entry,
classname))
# generate test suite result using provided test cases
test_suite = TestSuite(suitename, testcases)
# get rid of unnecessary 'disabled' strings in formatted xml string
s = TestSuite.to_xml_string([test_suite])
s = s.replace(' disabled=\"0\"', '')
# write xml to outfile
outfile.write(s)
def exporter_junit(test_result_ext, test_suite_properties=None):
"""! Export test results in JUnit XML compliant format
@details This function will import junit_xml library to perform report conversion
@return String containing Junit XML formatted test result output
"""
from junit_xml import TestSuite, TestCase
test_suites = []
test_cases = []
targets = sorted(test_result_ext.keys())
for target in targets:
test_cases = []
tests = sorted(test_result_ext[target].keys())
for test in tests:
test_results = test_result_ext[target][test]
classname = 'test.%s.%s' % (target, test)
elapsed_sec = test_results['elapsed_time']
_stdout = test_results['single_test_output']
_stderr = ''
# Test case
tc = TestCase(test, classname, elapsed_sec, _stdout, _stderr)
# Test case extra failure / error info
if test_results['single_test_result'] == 'FAIL':
message = test_results['single_test_result']
tc.add_failure_info(message, _stdout)
elif test_results['single_test_result'] != 'OK':
message = test_results['single_test_result']
tc.add_error_info(message, _stdout)
test_cases.append(tc)
ts = TestSuite("test.suite.%s" % target, test_cases)
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
def writeJUnitSLAContent(slas, test, filepath):
logger = logging.getLogger("root")
if slas is None: return
logger.info("writeJUnitSLAContent: " + test.id + " to " + filepath)
try:
indicators = slas['indicators']
perrun = slas['perrun']
perinterval = slas['perinterval']
suites = []
for sla in perrun:
ts = getSLATestSuites(test, "PerRun", sla)
suites.append(ts)
for sla in perinterval:
ts = getSLATestSuites(test, "PerInterval", sla)
suites.append(ts)
logger.debug(TestSuite.to_xml_string(suites))
with open(filepath, 'w') as f:
TestSuite.to_file(f, suites, prettyprint=True)
return True
except:
logger.error("Unexpected error at 'writeJUnitSLAContent':",
sys.exc_info()[0])
return False
def exporter_junit_ioper(self,
test_result_ext,
test_suite_properties=None):
from junit_xml import TestSuite, TestCase
test_suites = []
test_cases = []
for platform in sorted(test_result_ext.keys()):
# {platform : ['Platform', 'Result', 'Scope', 'Description'])
test_cases = []
for tr_result in test_result_ext[platform]:
result, name, scope, description = tr_result
classname = 'test.ioper.%s.%s.%s' % (platform, name, scope)
elapsed_sec = 0
_stdout = description
_stderr = ''
# Test case
tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
# Test case extra failure / error info
if result == 'FAIL':
tc.add_failure_info(description, _stdout)
elif result == 'ERROR':
tc.add_error_info(description, _stdout)
elif result == 'SKIP':
tc.add_skipped_info(description, _stdout)
test_cases.append(tc)
ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases)
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
def _write_junit_report(site, config, output_file, total_time):
pages = site.pages
test_cases = []
for results, resource in pages.items():
origins = [source.origin.geturl() for source in resource.sources]
if resource.status == 200:
test_case = TestCase(name=resource.url_split.geturl(),
classname=results.hostname,
elapsed_sec=resource.response_time,
stdout=resource.status,
status="passed")
else:
stderr_message = "Link found on:\n{}".format("\n".join(origins))
test_case = TestCase(name=resource.url_split.geturl(),
classname=results.hostname,
elapsed_sec=resource.response_time,
stderr=stderr_message,
status="failed")
if resource.exception:
message = str(resource.exception)
else:
message = "Expected 200 OK but got {}".format(resource.status)
test_case.add_failure_info(message=message,
failure_type="UnexpectedStatusCode")
test_cases.append(test_case)
test_suite = TestSuite("pylinkvalidator test suite", test_cases)
output_file.write(TestSuite.to_xml_string([test_suite]))
print_summary(site, config, total_time)
def parse(infile, outfile, format_type, classname, suitename):
testcases = list()
testcase_logs = list()
current = None
test_block_delimiter = known_formats[format_type]['tb_delimiter']
# separate log file into test blocks by test block delimiter
for line in infile:
if test_block_delimiter(line):
if current: # non-empty list
testcase_logs.append(current)
current = list()
if current is not None:
current.append(line)
# add last record if present
if current not in testcase_logs:
testcase_logs.append(current)
# create test cases from test blocks
for entry in testcase_logs:
testcases.append(known_formats[format_type]['test_parser'](entry, classname))
# generate test suite result using provided test cases
test_suite = TestSuite(suitename, testcases)
# get rid of unnecessary 'disabled' strings in formatted xml string
s = TestSuite.to_xml_string([test_suite])
s = s.replace(' disabled=\"0\"', '')
# write xml to outfile
outfile.write(s)
def print_result_cache_junitxml(dict_synonyms, suspicious_policy,
untested_policy):
test_cases = []
l = list(select(x for x in Mutant))
for filename, mutants in groupby(l,
key=lambda x: x.line.sourcefile.filename):
for mutant in mutants:
tc = TestCase("Mutant #{}".format(mutant.id),
file=filename,
line=mutant.line.line_number,
stdout=mutant.line.line)
if mutant.status == BAD_SURVIVED:
tc.add_failure_info(message=mutant.status,
output=get_unified_diff(
mutant.id, dict_synonyms))
if mutant.status == BAD_TIMEOUT:
tc.add_error_info(message=mutant.status,
error_type="timeout",
output=get_unified_diff(
mutant.id, dict_synonyms))
if mutant.status == OK_SUSPICIOUS:
if suspicious_policy != 'ignore':
func = getattr(tc, 'add_{}_info'.format(suspicious_policy))
func(message=mutant.status,
output=get_unified_diff(mutant.id, dict_synonyms))
if mutant.status == UNTESTED:
if untested_policy != 'ignore':
func = getattr(tc, 'add_{}_info'.format(untested_policy))
func(message=mutant.status,
output=get_unified_diff(mutant.id, dict_synonyms))
test_cases.append(tc)
ts = TestSuite("mutmut", test_cases)
print(TestSuite.to_xml_string([ts]))
def create_report(self,
report_file=None,
suite_name='Test Suite',
suite_package='root'):
"""
Create a test suite and write report to file specified
in report_file. Returns the resulting report string (or None if not
activated.)
"""
if self.activated:
if '.xml'.casefold() not in report_file.casefold():
report_file = report_file.split('.')[0] + '.xml'
ts = [
TestSuite(suite_name, self.test_cases, package=suite_package)
]
xmls = TestSuite.to_xml_string(ts, prettyprint=True)
if report_file:
if hasattr(report_file, 'write'):
# assume file-like
report_file.write(xmls)
else:
# Note: depending on system, mode may need to be set to wb
with open(report_file, mode='w') as f:
f.write(xmls)
print('Test report written to {}'.format(
os.path.abspath(f.name)))
return xmls
def generate_junit_report_from_cfn_nag(report):
total_failures = 0
"""Generate Test Case from cfn_nag report"""
test_cases = []
for file_findings in report:
for violation in file_findings["file_results"]['violations']:
total_failures += 1
for i, resource_id in enumerate(violation['logical_resource_ids']):
test_case = TestCase("%s - %s" %
(violation['id'], violation['message']),
classname=resource_id)
test_case.add_failure_info(
output="%s#L%s" %
(file_findings['filename'], violation['line_numbers'][i]))
test_cases.append(test_case)
test_suite = TestSuite("cfn-nag test suite", test_cases)
if total_failures > 0:
f = open("CFN_NAG_FAILURE", "a")
f.close()
return TestSuite.to_xml_string([test_suite], prettyprint=False)
def build_junit_xml_output(rule_matches: List[RuleMatch],
rules: FrozenSet[Rule]) -> str:
"""
Format matches in JUnit XML format.
"""
test_cases = [match.to_junit_xml() for match in rule_matches]
ts = TestSuite("semgrep results", test_cases)
return cast(str, TestSuite.to_xml_string([ts]))
def execute(self, log, keyvals, testDef):
testDef.logger.verbose_print("JunitXML Reporter")
# pickup the options
cmds = {}
testDef.parseOptions(log, self.options, keyvals, cmds)
if cmds['filename'] is not None:
self.fh = open(cmds['filename'] if os.path.isabs(cmds['filename']) \
else os.path.join(testDef.options['scratchdir'],cmds['filename']), 'w')
if testDef.options['description'] is not None:
print(testDef.options['description'], file=self.fh)
print(file=self.fh)
# Use the Junit classname field to store the list of inifiles
try:
classname = testDef.log['inifiles']
except KeyError:
classname = None
# get the entire log of results
fullLog = testDef.logger.getLog(None)
testCases = []
# TODO: ain't nobody got time for that. 8-).
time = 0
for lg in fullLog:
if 'stdout' in lg and lg['stdout'] is not None:
stdout = "\n".join(lg['stdout'])
else:
stdout = None
if 'stderr' in lg and lg['stderr'] is not None:
stderr = "\n".join(lg['stderr'])
else:
stderr = None
if 'time' in lg and lg['time'] is not None:
time = lg['time']
else:
time = 0
tc = TestCase(lg['section'], classname, time, stdout, stderr)
try:
if 0 != lg['status']:
# Find sections prefixed with 'TestRun'
if re.match("TestRun", lg['section']):
tc.add_failure_info("Test reported failure")
else:
tc.add_error_info("Test error")
except KeyError:
sys.exit(lg['section'] + " is missing status!")
testCases.append(tc)
# TODO: Pull in the resource manager jobid.
jobid = "job1"
ts = TestSuite(jobid, testCases)
print(TestSuite.to_xml_string([ts]), file=self.fh)
if cmds['filename'] is not None:
self.fh.close()
log['status'] = 0
return
def execute(self, log, keyvals, testDef):
testDef.logger.verbose_print("JunitXML Reporter")
# pickup the options
cmds = {}
testDef.parseOptions(log, self.options, keyvals, cmds)
if cmds['filename'] is not None:
self.fh = open(cmds['filename'] if os.path.isabs(cmds['filename']) \
else os.path.join(cmds['scratch'],cmds['filename']), 'w')
if testDef.options['description'] is not None:
print(testDef.options['description'], file=self.fh)
print(file=self.fh)
# Use the Junit classname field to store the list of inifiles
try:
classname = testDef.log['inifiles']
except KeyError:
classname = None
# get the entire log of results
fullLog = testDef.logger.getLog(None)
testCases = []
# TODO: ain't nobody got time for that. 8-).
time = 0
for lg in fullLog:
if 'stdout' in lg and lg['stdout'] is not None:
stdout = "\n".join(lg['stdout'])
else:
stdout = None
if 'stderr' in lg and lg['stderr'] is not None:
stderr = "\n".join(lg['stderr'])
else:
stderr = None
if 'time' in lg and lg['time'] is not None:
time = lg['time']
else:
time = 0
tc = TestCase(lg['section'], classname, time, stdout, stderr)
try:
if 0 != lg['status']:
# Find sections prefixed with 'TestRun'
if re.match("TestRun", lg['section']):
tc.add_failure_info("Test reported failure")
else:
tc.add_error_info("Test error")
except KeyError:
sys.exit(lg['section'] + " is missing status!")
testCases.append(tc)
# TODO: Pull in the resource manager jobid.
jobid = "job1"
ts = TestSuite(jobid, testCases)
print(TestSuite.to_xml_string([ts]), file=self.fh)
if cmds['filename'] is not None:
self.fh.close()
log['status'] = 0
return
def run_api_tests(args, data_format):
endpoints = []
for i in range(len(args.host)):
endpoints.append({"host": args.host[i], "port": args.port[i], "version": args.version[i]})
results = run_tests(args.suite, endpoints, [args.selection])
if data_format == "xml":
formatted_test_results = format_test_results(results, endpoints, "junit", args)
return TestSuite.to_xml_string([formatted_test_results], prettyprint=True)
else:
formatted_test_results = format_test_results(results, endpoints, "json", args)
return json.loads(formatted_test_results)
def exporter_junit(self, test_result_ext, test_suite_properties=None):
""" Export test results in JUnit XML compliant format
"""
from junit_xml import TestSuite, TestCase
test_suites = []
test_cases = []
targets = sorted(test_result_ext.keys())
for target in targets:
toolchains = sorted(test_result_ext[target].keys())
for toolchain in toolchains:
test_cases = []
tests = sorted(test_result_ext[target][toolchain].keys())
for test in tests:
test_results = test_result_ext[target][toolchain][test]
for test_res in test_results:
test_ids = sorted(test_res.keys())
for test_no in test_ids:
test_result = test_res[test_no]
name = test_result['description']
classname = '%s.%s.%s.%s' % (self.package, target,
toolchain,
test_result['id'])
elapsed_sec = test_result['elapsed_time']
_stdout = test_result['output']
if 'target_name_unique' in test_result:
_stderr = test_result['target_name_unique']
else:
_stderr = test_result['target_name']
# Test case
tc = TestCase(name, classname, elapsed_sec,
_stdout, _stderr)
# Test case extra failure / error info
message = test_result['result']
if test_result['result'] == 'FAIL':
tc.add_failure_info(message, _stdout)
elif test_result['result'] == 'SKIP' or test_result[
"result"] == 'NOT_SUPPORTED':
tc.add_skipped_info(message, _stdout)
elif test_result['result'] != 'OK':
tc.add_error_info(message, _stdout)
test_cases.append(tc)
ts = TestSuite(
"test.suite.%s.%s" % (target, toolchain),
test_cases,
properties=test_suite_properties[target][toolchain])
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
def take_action(self, args):
test_cases = []
if args.playbook is not None:
playbooks = args.playbook
results = (models.TaskResult().query
.join(models.Task)
.filter(models.TaskResult.task_id == models.Task.id)
.filter(models.Task.playbook_id.in_(playbooks)))
else:
results = models.TaskResult().query.all()
for result in results:
task_name = result.task.name
if not task_name:
task_name = result.task.action
additional_results = {
'host': result.host.name,
'playbook_path': result.task.playbook.path
}
result_str = jsonutils.dumps(additional_results)
test_path = \
u'{playbook_file}.{play_name}'.format(
playbook_file=os.path.basename(result.task.playbook.path),
play_name=result.task.play.name)
test_case = TestCase(
name=task_name,
classname=test_path,
elapsed_sec=result.duration.seconds,
stdout=result_str)
if result.status == 'skipped':
test_case.add_skipped_info(message=result.result)
elif ((result.status in ('failed', 'unreachable') and
result.ignore_errors is False and
'EXPECTED FAILURE' not in task_name and
'TOGGLE RESULT' not in task_name) or
(result.status == 'ok' and 'TOGGLE RESULT' in task_name)):
test_case.add_failure_info(message=result.result)
test_cases.append(test_case)
test_suite = TestSuite('Ansible Tasks', test_cases)
# TODO: junit_xml doesn't order the TestCase parameters.
# This makes it so the order of the parameters for the same exact
# TestCase is not guaranteed to be the same and thus results in a
# different stdout (or file). This is easily reproducible on Py3.
xml_string = six.text_type(test_suite.to_xml_string([test_suite]))
if args.output_file == '-':
if six.PY2:
sys.stdout.write(encodeutils.safe_encode(xml_string))
else:
sys.stdout.buffer.write(encodeutils.safe_encode(xml_string))
else:
with open(args.output_file, 'wb') as f:
f.write(encodeutils.safe_encode(xml_string))
def get_junit_xml(res):
# notes = res.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_WARNING)
notes = res.get_notes_by_tag(MCDPManualConstants.NOTE_TAG_ERROR)
test_cases = []
for i, note in enumerate(notes):
tc = junit_test_case_from_note(i, note)
test_cases.append(tc)
ts = TestSuite("notes", test_cases)
return TestSuite.to_xml_string([ts])
def exporter_junit(self, test_result_ext, test_suite_properties=None):
""" Export test results in JUnit XML compliant format
"""
from junit_xml import TestSuite, TestCase
test_suites = []
test_cases = []
targets = sorted(test_result_ext.keys())
for target in targets:
toolchains = sorted(test_result_ext[target].keys())
for toolchain in toolchains:
test_cases = []
tests = sorted(test_result_ext[target][toolchain].keys())
for test in tests:
test_results = test_result_ext[target][toolchain][test]
for test_res in test_results:
test_ids = sorted(test_res.keys())
for test_no in test_ids:
test_result = test_res[test_no]
name = test_result["description"]
classname = "%s.%s.%s.%s" % (self.package, target, toolchain, test_result["id"])
elapsed_sec = test_result["elapsed_time"]
_stdout = test_result["output"]
if "target_name_unique" in test_result:
_stderr = test_result["target_name_unique"]
else:
_stderr = test_result["target_name"]
# Test case
tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
# Test case extra failure / error info
message = test_result["result"]
if test_result["result"] == "FAIL":
tc.add_failure_info(message, _stdout)
elif test_result["result"] == "SKIP" or test_result["result"] == "NOT_SUPPORTED":
tc.add_skipped_info(message, _stdout)
elif test_result["result"] != "OK":
tc.add_error_info(message, _stdout)
test_cases.append(tc)
ts = TestSuite(
"test.suite.%s.%s" % (target, toolchain),
test_cases,
properties=test_suite_properties[target][toolchain],
)
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
def execute(self, log, keyvals, testDef):
testDef.logger.verbose_print("JunitXML Reporter")
# pickup the options
cmds = {}
testDef.parseOptions(log, self.options, keyvals, cmds)
if cmds['filename'] is not None:
self.fh = open(cmds['filename'] if os.path.isabs(cmds['filename']) \
else os.path.join(testDef.options['scratchdir'],cmds['filename']), 'w')
# get the entire log of results
fullLog = testDef.logger.getLog(None)
testCases = []
time = 0
for lg in fullLog:
if 'stdout' in lg and lg['stdout'] is not None:
stdout = "\n".join(lg['stdout'])
else:
stdout = None
if 'stderr' in lg and lg['stderr'] is not None:
stderr = "\n".join(lg['stderr'])
else:
stderr = None
if 'time' in lg and lg['time'] is not None:
time = lg['time']
else:
time = 0
# Use the hostname of the system we are running on as root of the classname
# Use the filename without the extension as the next layer of the classname
hostname = os.uname()[1]
classname = hostname + "." + cmds['filename'].split('.')[0]
tc = TestCase(lg['section'], classname, time, stdout, stderr)
try:
if 0 != lg['status']:
# Find sections prefixed with 'TestRun'
if re.match("TestRun", lg['section']):
tc.add_failure_info("Test reported failure")
else:
tc.add_error_info("Test error")
except KeyError:
sys.exit(lg['section'] + " is missing status!")
testCases.append(tc)
# TODO: Pull in the resource manager jobid.
jobid = "job1"
ts = TestSuite(jobid, testCases)
print(TestSuite.to_xml_string([ts]), file=self.fh)
if cmds['filename'] is not None:
self.fh.close()
log['status'] = 0
return
def take_action(self, args):
test_cases = []
if args.playbook is not None:
playbooks = args.playbook
results = (models.TaskResult().query.join(models.Task).filter(
models.TaskResult.task_id == models.Task.id).filter(
models.Task.playbook_id.in_(playbooks)))
else:
results = models.TaskResult().query.all()
for result in results:
task_name = result.task.name
if not task_name:
task_name = result.task.action
additional_results = {
'host': result.host.name,
'playbook_path': result.task.playbook.path
}
result_str = jsonutils.dumps(additional_results)
test_path = \
u'{playbook_file}.{play_name}'.format(
playbook_file=os.path.basename(result.task.playbook.path),
play_name=result.task.play.name)
test_case = TestCase(name=task_name,
classname=test_path,
elapsed_sec=result.duration.seconds,
stdout=result_str)
if result.status == 'skipped':
test_case.add_skipped_info(message=result.result)
elif ((result.status in ('failed', 'unreachable')
and result.ignore_errors is False
and 'EXPECTED FAILURE' not in task_name
and 'TOGGLE RESULT' not in task_name)
or (result.status == 'ok' and 'TOGGLE RESULT' in task_name)):
test_case.add_failure_info(message=result.result)
test_cases.append(test_case)
test_suite = TestSuite('Ansible Tasks', test_cases)
# TODO: junit_xml doesn't order the TestCase parameters.
# This makes it so the order of the parameters for the same exact
# TestCase is not guaranteed to be the same and thus results in a
# different stdout (or file). This is easily reproducible on Py3.
xml_string = six.text_type(test_suite.to_xml_string([test_suite]))
if args.output_file == '-':
if six.PY2:
sys.stdout.write(encodeutils.safe_encode(xml_string))
else:
sys.stdout.buffer.write(encodeutils.safe_encode(xml_string))
else:
with open(args.output_file, 'wb') as f:
f.write(encodeutils.safe_encode(xml_string))
def serialize_and_read(test_suites,
to_file=False,
prettyprint=False,
encoding=None):
"""writes the test suite to an XML string and then re-reads it using minidom,
returning => (test suite element, list of test case elements)"""
try:
iter(test_suites)
except TypeError:
test_suites = [test_suites]
if to_file:
fd, filename = tempfile.mkstemp(text=True)
os.close(fd)
with codecs.open(filename, mode='w', encoding=encoding) as f:
TestSuite.to_file(f,
test_suites,
prettyprint=prettyprint,
encoding=encoding)
print("Serialized XML to temp file [%s]" % filename)
xmldoc = minidom.parse(filename)
os.remove(filename)
else:
xml_string = TestSuite.to_xml_string(test_suites,
prettyprint=prettyprint,
encoding=encoding)
if PY2:
assert isinstance(xml_string, unicode)
print("Serialized XML to string:\n%s" % xml_string)
if encoding:
xml_string = xml_string.encode(encoding)
xmldoc = minidom.parseString(xml_string)
def remove_blanks(node):
for x in node.childNodes:
if x.nodeType == minidom.Node.TEXT_NODE:
if x.nodeValue:
x.nodeValue = x.nodeValue.strip()
elif x.nodeType == minidom.Node.ELEMENT_NODE:
remove_blanks(x)
remove_blanks(xmldoc)
xmldoc.normalize()
ret = []
suites = xmldoc.getElementsByTagName("testsuites")[0]
for suite in suites.getElementsByTagName("testsuite"):
cases = suite.getElementsByTagName("testcase")
ret.append((suite, cases))
return ret
def process_to_xml(process, test_name, suite_name):
t0 = time()
stdout, stderr = process.communicate()
duration = time() - t0
tc = TestCase(
name=test_name,
elapsed_sec=duration,
stdout=stdout.decode(),
stderr=stderr.decode(),
)
if process.returncode != 0:
tc.add_failure_info(stderr.decode())
return TestSuite.to_xml_string(
[TestSuite(name=suite_name, test_cases=[tc])])
def create_junit_results(data, output=None, **kwargs):
"""
Creates a Junit result, can write to a file if desired, or return xml string. (used by Jenkins)
input either dict(dict(dict())) or dict(list(dict()))
dict = {suite: {test: {stderr,stdout,time,class,err,fail,skip}}}
list = {suite: [(test, {stderr,stdout,time,class,err,fail,skip})]}
:param data: A dictionary with dict or list hierarchy
:param output: A filename to write results to /path/to/file/*.junit.xml
:return: Returns an XML string if no output, else nothing.
"""
log.debug('creating junit results: output={}'.format(output))
stdout_format = kwargs.pop('stdout_format', None)
test_class = kwargs.pop('test_class', None)
package = kwargs.pop('package', None)
from junit_xml import TestSuite, TestCase
test_suites = []
for suite, tests in data.items():
test_cases = []
for test, result in (tests if isinstance(tests, list) else tests.items()):
tc = TestCase(test)
stdout = result.get('stdout')
if stdout_format is not None and callable(stdout_format):
if hasattr(stdout_format, 'func_code') and 'kwargs' in stdout_format.func_code.co_varnames:
stdout = stdout_format(stdout, suite_name=suite, test_name=test, **kwargs)
else:
stdout = stdout_format(stdout)
tc.stdout = stdout
tc.stderr = result.get('stderr')
tc.elapsed_sec = result.get('time')
tc.classname = result.get('class', test_class)
err = result.get('err')
if err:
tc.add_error_info(*err if isinstance(err, (list, tuple)) else [err])
fail = result.get('fail')
if fail:
tc.add_failure_info(*fail if isinstance(fail, (list, tuple)) else [fail])
skip = result.get('skip')
if skip:
tc.add_skipped_info(*skip if isinstance(skip, (list, tuple)) else [skip])
test_cases.append(tc)
ts = TestSuite(suite, test_cases, package=package)
test_suites.append(ts)
if output:
check_makedir(os.path.dirname(output))
with open(output, 'w') as out:
TestSuite.to_file(out, test_suites)
return output
else:
return TestSuite.to_xml_string(test_suites)
def junit_xml(compmake_db):
jobs = list(all_jobs(compmake_db))
logger.info('Loaded %d jobs' % len(jobs))
if len(jobs) < 10:
logger.error('too few jobs')
sys.exit(128)
test_cases = []
for job_id in jobs:
tc = junit_test_case_from_compmake(compmake_db, job_id)
test_cases.append(tc)
ts = TestSuite("comptests_test_suite", test_cases)
return TestSuite.to_xml_string([ts])
def show(self):
measurement_data = "<measurement><name>Events</name><value>%s</value></measurement>" % (
self.event_count)
monkey_test = TestCase('monkey', stdout=measurement_data)
# handle failure msg
if (self.is_success is False):
monkey_test.add_failure_info(self.failure_msg)
ts = TestSuite("com.skysoft.kkbox.android", [monkey_test])
# pretty printing is on by default but can be disabled using prettyprint=False
junit_data = TestSuite.to_xml_string([ts])
print(junit_data)
def _generate_report(self):
""" generate a TestSuite report from the collected TaskData and HostData """
test_cases = []
for task_uuid, task_data in self._task_data.items():
for host_uuid, host_data in task_data.host_data.items():
test_cases.append(self._build_test_case(task_data, host_data))
test_suite = TestSuite(self._playbook_name, test_cases)
report = TestSuite.to_xml_string([test_suite])
output_file = os.path.join(self._output_dir, '%s-%s.xml' % (self._playbook_name, time.time()))
with open(output_file, 'wb') as xml:
xml.write(to_bytes(report, errors='surrogate_or_strict'))
def test_to_xml_string(self):
test_suites = [TestSuite('suite1', [TestCase('Test1')]),
TestSuite('suite2', [TestCase('Test2')])]
xml_string = TestSuite.to_xml_string(test_suites)
expected_xml_string = textwrap.dedent("""
<?xml version="1.0" ?>
<testsuites>
\t<testsuite errors="0" failures="0" name="suite1" skipped="0" tests="1" time="0">
\t\t<testcase name="Test1"/>
\t</testsuite>
\t<testsuite errors="0" failures="0" name="suite2" skipped="0" tests="1" time="0">
\t\t<testcase name="Test2"/>
\t</testsuite>
</testsuites>
""".strip("\n"))
self.assertEqual(xml_string, expected_xml_string)
def _generate_report(self):
""" generate a TestSuite report from the collected TaskData and HostData """
test_cases = []
for task_uuid, task_data in self._task_data.items():
for host_uuid, host_data in task_data.host_data.items():
test_cases.append(self._build_test_case(task_data, host_data))
test_suite = TestSuite(self._playbook_name, test_cases)
report = TestSuite.to_xml_string([test_suite])
output_file = os.path.join(self._output_dir, '%s-%s.xml' % (self._playbook_name, time.time()))
with open(output_file, 'wb') as xml:
xml.write(to_bytes(report, errors='strict'))
def exporter_junit(self, test_result_ext, test_suite_properties=None):
""" Export test results in JUnit XML compliant format
"""
from junit_xml import TestSuite, TestCase
test_suites = []
test_cases = []
targets = sorted(test_result_ext.keys())
for target in targets:
toolchains = sorted(test_result_ext[target].keys())
for toolchain in toolchains:
test_cases = []
tests = sorted(test_result_ext[target][toolchain].keys())
for test in tests:
test_results = test_result_ext[target][toolchain][test]
for test_res in test_results:
test_ids = sorted(test_res.keys())
for test_no in test_ids:
test_result = test_res[test_no]
name = test_result['description']
classname = '%s.%s.%s.%s'% (self.package, target, toolchain, test_result['id'])
elapsed_sec = test_result['elapsed_time']
_stdout = test_result['output']
if 'target_name_unique' in test_result:
_stderr = test_result['target_name_unique']
else:
_stderr = test_result['target_name']
# Test case
tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
# Test case extra failure / error info
message = test_result['result']
if test_result['result'] == 'FAIL':
tc.add_failure_info(message, _stdout)
elif test_result['result'] == 'SKIP' or test_result["result"] == 'NOT_SUPPORTED':
tc.add_skipped_info(message, _stdout)
elif test_result['result'] != 'OK':
tc.add_error_info(message, _stdout)
test_cases.append(tc)
ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain])
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
def _test():
test_case1 = TestCase('Testname1', 'SetupCluster.Name')
test_case2 = TestCase('Testname2', 'SetupCluster.Name')
test_case3 = TestCase('Testname3', 'SetupCluster.Misc')
test_case4 = TestCase('Testname4', 'SetupCluster.Misc')
test_cases = [test_case1, test_case2, test_case3, test_case4]
ts = TestSuite("My Test Suite", test_cases)
#Run and verify test case
assertCase(test_case1, _exampleFunc(True))
assertCase(test_case2, _exampleFunc(False))
assertCase(test_case3, _exampleFunc(True))
#Skip test case
skipCase(test_case4, "Skip Testname4.", "Testname2 is failed.")
print(ts.to_xml_string([ts]))
def test_to_xml_string(self):
test_suites = [
TestSuite('suite1', [TestCase('Test1')]),
TestSuite('suite2', [TestCase('Test2')])
]
xml_string = TestSuite.to_xml_string(test_suites)
expected_xml_string = textwrap.dedent("""
<?xml version="1.0" ?>
<testsuites>
\t<testsuite errors="0" failures="0" name="suite1" skipped="0" tests="1" time="0">
\t\t<testcase name="Test1"/>
\t</testsuite>
\t<testsuite errors="0" failures="0" name="suite2" skipped="0" tests="1" time="0">
\t\t<testcase name="Test2"/>
\t</testsuite>
</testsuites>
""".strip("\n"))
self.assertEqual(xml_string, expected_xml_string)
def test_to_xml_string(self):
test_suites = [TestSuite(name='suite1', test_cases=[TestCase(name='Test1')]),
TestSuite(name='suite2', test_cases=[TestCase(name='Test2')])]
xml_string = TestSuite.to_xml_string(test_suites)
if PY2:
self.assertTrue(isinstance(xml_string, unicode))
expected_xml_string = textwrap.dedent("""
<?xml version="1.0" ?>
<testsuites disabled="0" errors="0" failures="0" tests="2" time="0.0">
\t<testsuite disabled="0" errors="0" failures="0" name="suite1" skipped="0" tests="1" time="0">
\t\t<testcase name="Test1"/>
\t</testsuite>
\t<testsuite disabled="0" errors="0" failures="0" name="suite2" skipped="0" tests="1" time="0">
\t\t<testcase name="Test2"/>
\t</testsuite>
</testsuites>
""".strip("\n")) # NOQA
self.assertEqual(xml_string, expected_xml_string)
def take_action(self, args):
test_cases = []
if args.playbook is not None:
playbooks = args.playbook
results = (models.TaskResult().query
.join(models.Task)
.filter(models.TaskResult.task_id == models.Task.id)
.filter(models.Task.playbook_id.in_(playbooks)))
else:
results = models.TaskResult().query.all()
for result in results:
task_name = result.task.name
if not task_name:
task_name = result.task.action
additional_results = {
'host': result.host.name,
'playbook_path': result.task.playbook.path
}
result_str = json.dumps(additional_results)
test_path = \
"{playbook_file}.{play_name}".format(
playbook_file=os.path.basename(result.task.playbook.path),
play_name=result.task.play.name)
test_case = TestCase(
name=task_name,
classname=test_path,
elapsed_sec=result.duration.seconds,
stdout=result_str)
if result.status == "skipped":
test_case.add_skipped_info(message=result.result)
elif (result.status in ("failed", "unreachable") and
result.ignore_errors is False):
test_case.add_failure_info(message=result.result)
test_cases.append(test_case)
test_suite = TestSuite("Ansible Tasks", test_cases)
xml_string = test_suite.to_xml_string([test_suite])
if args.output_file == "-":
sys.stdout.write(xml_string)
else:
with open(args.output_file, "w") as f:
f.write(xml_string)
def main():
# Read list of html files in from stdin line by line.
# This is to enable piping result from `find`
# Use a multiprocessing pool to get some parallelism
pool = Pool()
queries = []
test_cases = []
for line in sys.stdin:
queries.append(pool.apply_async(run_test, [line.strip()]))
for query in queries:
test_cases.append(query.get(timeout=10))
# Dump test results to stdout
ts = [TestSuite("my test suite", test_cases)]
print(TestSuite.to_xml_string(ts))
return 0
def serialize_and_read(test_suites, to_file=False, prettyprint=False, encoding=None):
"""writes the test suite to an XML string and then re-reads it using minidom,
returning => (test suite element, list of test case elements)"""
try:
iter(test_suites)
except TypeError:
test_suites = [test_suites]
if to_file:
fd, filename = tempfile.mkstemp(text=True)
os.close(fd)
with codecs.open(filename, mode='w', encoding=encoding) as f:
TestSuite.to_file(f, test_suites, prettyprint=prettyprint, encoding=encoding)
print("Serialized XML to temp file [%s]" % filename)
xmldoc = minidom.parse(filename)
os.remove(filename)
else:
xml_string = TestSuite.to_xml_string(
test_suites, prettyprint=prettyprint, encoding=encoding)
if PY2:
assert isinstance(xml_string, unicode)
print("Serialized XML to string:\n%s" % xml_string)
if encoding:
xml_string = xml_string.encode(encoding)
xmldoc = minidom.parseString(xml_string)
def remove_blanks(node):
for x in node.childNodes:
if x.nodeType == minidom.Node.TEXT_NODE:
if x.nodeValue:
x.nodeValue = x.nodeValue.strip()
elif x.nodeType == minidom.Node.ELEMENT_NODE:
remove_blanks(x)
remove_blanks(xmldoc)
xmldoc.normalize()
ret = []
suites = xmldoc.getElementsByTagName("testsuites")[0]
for suite in suites.getElementsByTagName("testsuite"):
cases = suite.getElementsByTagName("testcase")
ret.append((suite, cases))
return ret
def junit_xml(compmake_db):
from junit_xml import TestSuite
jobs = list(all_jobs(compmake_db))
logger.info('Loaded %d jobs' % len(jobs))
N = 10
if len(jobs) < N:
logger.error('too few jobs (I expect at least %s)' % N)
sys.exit(128)
test_cases = []
for job_id in jobs:
tc = junit_test_case_from_compmake(compmake_db, job_id)
test_cases.append(tc)
ts = TestSuite("comptests_test_suite", test_cases)
res = TestSuite.to_xml_string([ts])
check_isinstance(res, six.text_type)
return res
def xmlwrite(target, everyline):
xmltargetpath = os.getcwd() + os.sep + target + '_testresults.xml'
xmltargetfile = open(xmltargetpath, 'w+')
test_cases = []
for line in everyline.split('\n'):
words = line.split(' ')
if line.find('success') != -1:
time_taken = get_time(line.split('-')[-1][7:])
name = words[0] + ' - ' + line.split('-')[1].split('name:')[1].strip()
print name
test_cases.append(TestCase(name, target, time_taken, None))
elif line.find('failed') != -1:
time_taken = get_time(line.split('-')[-1][7:])
name = words[0] + ' - ' + line.split('-')[1].split('name:')[1].strip()
message = ("-".join(line.split(' - ')[2:-1])).strip()
tc = TestCase(name, target, time_taken, None)
tc.add_failure_info(None, message)
test_cases.append(tc)
ts = TestSuite("testing this suite", test_cases)
xmltargetfile.write(TestSuite.to_xml_string([ts]))
xmltargetfile.close()
def report(self, report_dir=None):
test_cases = []
if report_dir is None:
report_dir = self.config.report_dir
for report_file in os.listdir(report_dir):
if report_file.endswith(".pkl"):
f = open(os.path.join(report_dir, report_file), "r")
result_dict = cPickle.load(f)
f.close()
tests = result_dict.keys()
tests.sort()
for test in tests:
in_entry = result_dict[test]
report_entry = TestCase(test, in_entry["CLASS_NAME"], in_entry["TIME"])
if in_entry["RESULT"] == "FAIL":
report_entry.add_failure_info(in_entry["MESSAGE"], in_entry["TRACE"])
elif in_entry["RESULT"] == "ERROR":
report_entry.add_error_info(in_entry["MESSAGE"], in_entry["TRACE"])
test_cases.append(report_entry)
ts = TestSuite("my test suite", test_cases)
f_xml = open(os.path.join(report_dir, "results.xml"), "w")
f_xml.write(TestSuite.to_xml_string([ts]))
f_xml.close()
def exporter_junit(self, test_result_ext, test_suite_properties=None):
""" Export test results in JUnit XML compliant format
"""
from junit_xml import TestSuite, TestCase
test_suites = []
test_cases = []
toolchains = sorted(test_result_ext.keys())
for toolchain in toolchains:
targets = sorted(test_result_ext[toolchain].keys())
for target in targets:
test_cases = []
tests = sorted(test_result_ext[toolchain][target].keys())
for test in tests:
test_results = test_result_ext[toolchain][target][test]
test_ids = sorted(test_results.keys())
for test_no in test_ids:
test_result = test_results[test_no]
name = test_result['test_description']
classname = 'test.%s.%s.%s'% (target, toolchain, test_result['test_id'])
elapsed_sec = test_result['elapsed_time']
_stdout = test_result['single_test_output']
_stderr = ''
# Test case
tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
# Test case extra failure / error info
if test_result['single_test_result'] == 'FAIL':
message = test_result['single_test_result']
tc.add_failure_info(message, _stdout)
elif test_result['single_test_result'] != 'OK':
message = test_result['single_test_result']
tc.add_error_info(message, _stdout)
test_cases.append(tc)
ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain])
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
def failureSimulation(self,failure_interval, wait_before_start, servers, min_servers, servers_to_fail_simultaneously, kill_method, initial_clean, junit_report):
"""
Run the failure loop for a given role
"""
if(servers == ""):
print "--servers not specified!\n\n"
return
logging.debug("Failure interval: " + str(failure_interval))
logging.debug("Wait before start: " + str(wait_before_start))
logging.debug("Server list: " + servers)
logging.debug("Minimum number of servers: " + str(min_servers))
logging.debug("Number of servers to fail simultaneously: " + str(servers_to_fail_simultaneously))
logging.debug("Kill method: " + kill_method)
logging.debug("Initial clean: " + str(initial_clean))
logging.debug("Role name: "+ self.roleName)
logging.debug("Junit Report: "+ junit_report)
testCases = []
testNum = 0
cluster = Cluster()
serverArray = servers.split(",")
cluster = cluster.getServersByHostname(serverArray)
if min_servers >= cluster.getNumServers():
raise ValueError("Minimum Number of servers is too high!\nMinimum Servers to stay up: "
+str(min_servers)+"\nNumber of "+self.roleName+" servers in cluster: "+str(cluster.getNumServers()))
exit(-1)
if servers_to_fail_simultaneously > cluster.getNumServers() - min_servers:
raise ValueError("--servers_to_fail_simultaneously is set too high")
exit(-1)
if initial_clean:
cluster.cleanProcess(self.roleName)
while True:
start = time()
logging.debug("Sleeping for "+str(failure_interval)+" seconds")
sleep(failure_interval)
#pick random servers to kill
serversToKill = sample(serverArray, servers_to_fail_simultaneously)
logging.debug("Servers selected to kill: "+ ','.join(serversToKill))
#Stop the running process based on kill_method
for hostname in serversToKill:
if kill_method == "restart" :
logging.debug("Shutting down "+self.roleName + " on " +hostname)
cluster.shutdownProcessOnHost(self.roleName, hostname)
elif kill_method == "kill":
logging.debug("Killing "+self.roleName + " on " +hostname)
cluster.killProcessOnHost(self.roleName, hostname)
else:
if randint(0,1) == 0:
logging.debug("Shutting down "+self.roleName + " on " +hostname)
cluster.shutdownProcessOnHost(self.roleName, hostname)
else:
logging.debug("Killing "+self.roleName + " on " +hostname)
cluster.killProcessOnHost(self.roleName, hostname)
#Ensure the process has stopped
for hostname in serversToKill:
#Create basis for test case
tc = TestCase('Test'+str(testNum), self.roleName+'FailureSimulator', time()-start,
'Shutting down '+self.roleName+" with kill_method "+kill_method+" on host "+hostname, '')
#If the process is still running, then try killing it one more time
if(cluster.isProcessRunningOnHost(self.roleName, hostname)):
logging.debug("Killing "+self.roleName + " on " +hostname+" one last time")
cluster.killProcessOnHost(self.roleName, hostname)
#If the process is *still* running then report a failure
if(cluster.isProcessRunningOnHost(self.roleName, hostname)):
tc.add_failure_info(self.roleName+" process is still running on"+hostname, "")
testCases.append(tc)
testNum+=1
#Start the process again
start = time()
sleep(wait_before_start)
for hostname in serversToKill:
logging.debug("Starting " + self.roleName + " on " + hostname)
cluster.startProcessOnHost(self.roleName, hostname)
#Ensure the process has started, otherwise report a failure
for hostname in serversToKill:
tc = TestCase('Test'+str(testNum), self.roleName+'FailureSimulator', time()-start,
'Starting '+self.roleName+" on host: "+hostname, '')
if( not cluster.isProcessRunningOnHost(self.roleName, hostname)):
tc.add_failure_info(self.roleName+" process is still running on"+hostname, "")
testCases.append(tc)
testNum+=1
if(not junit_report == "" ):
logging.debug("Writing junit report to: "+junit_report)
if not os.path.exists(junit_report):
os.makedirs(junit_report)
f = open(junit_report,'w')
ts = TestSuite(self.roleName+" Test Suite", testCases)
f.write(TestSuite.to_xml_string([ts]))
f.close()
from junit_xml import TestSuite, TestCase
test_cases1 = [TestCase('Test1', 'some.class.name', 123.345, 'I am stdout!', 'I am stderr!')]
test_cases2 = [TestCase('Test2', 'some.class.name', 123.345, 'I am stdout!', 'I am stderr!')]
test_cases3 = [TestCase('Test3', 'some.class.name', 'abc', 'I am stdout!', 'I am stderr!')]
ts1 = TestSuite("my test suite", test_cases1)
ts2 = TestSuite("my test suite", test_cases2)
ts3 = TestSuite("my test suite", test_cases3)
# pretty printing is on by default but can be disabled using prettyprint=False
print(TestSuite.to_xml_string([ts1]))
print(TestSuite.to_xml_string([ts2]))
print(TestSuite.to_xml_string([ts3]))
# you can also write the XML to a file and not pretty print it
#print(TestSuite.to_xml_string([ts]))
with open('output1.xml', 'w') as f1:
TestSuite.to_file(f1, [ts1], prettyprint=False)
with open('output2.xml', 'w') as f2:
TestSuite.to_file(f2, [ts2], prettyprint=False)
#with open('output3.xml', 'w') as f3:
# ## TestSuite.to_file(f3, [ts3], prettyprint=False)
from junit_xml import TestCase, TestSuite
caso_prueba = [TestCase('Prueba1', 'some.class.name', 123.345, 'Soy una salida', 'Soy un error')]
ts = TestSuite("Mi caso de prueba", caso_prueba)
print(TestSuite.to_xml_string([ts]))
>>> Running case #1: 'manifest'...
>>> 'manifest': 1 passed, 0 failed
>>> Running case #2: 'test_manifest_fragment'...
>>> Running case #3: 'test_firware_fragment'...
>>> 'test_firware_fragment': 1 passed, 0 failed
>>> Running case #4: 'test_keytable'...
dummy output
>>> 'test_keytable': 0 passed, 1 failed
>>> Test cases: 4 passed, 0 failed
"""
print TestSuite.to_xml_string([TestSuite("name", parse_result(result))])
result = \
"""
{{__testcase_count;4}}
>>> Running 4 test cases...
>>> Running case #1: 'simple_get_hash'...
{{__testcase_start;simple_get_hash}}
>>> Running case #2: 'simple_get_date'...
{{__testcase_start;simple_get_date}}
{{__testcase_finish;simple_get_date;1;0}}
>>> 'simple_get_date': 1 passed, 0 failed
>>> Running case #3: 'simple_get_fragment'...
def __init__ (self,
cmd_list,
nprocs=1,
out=sys.stdout,
log=None,
verbosity=DEFAULT_VERBOSITY,
output_junit_xml=False) :
if (log is None) : log = null_out()
self.out = multi_out()
self.log = log
self.out.register("stdout", out)
self.out.register("log", log)
self.verbosity = verbosity
self.quiet = (verbosity == 0)
self.results = []
# Filter cmd list for duplicates.
self.cmd_list = []
for cmd in cmd_list :
if (not cmd in self.cmd_list) :
self.cmd_list.append(cmd)
else :
print >> self.out, "Test %s repeated, skipping"%cmd
# Set number of processors.
if (nprocs is Auto) :
nprocs = cpu_count()
nprocs = min(nprocs, len(self.cmd_list))
# Starting summary.
if (self.verbosity > 0) :
print >> self.out, "Running %d tests on %s processors:"%(len(self.cmd_list), nprocs)
for cmd in self.cmd_list:
print >> self.out, " %s"%cmd
print >> self.out, ""
t_start = time.time()
if nprocs > 1:
# Run the tests with multiprocessing pool.
pool = Pool(processes=nprocs)
for command in self.cmd_list:
pool.apply_async(
run_command,
[command, verbosity, out],
callback=self.save_result)
try:
pool.close()
except KeyboardInterrupt:
print >> self.out, "Caught KeyboardInterrupt, terminating"
pool.terminate()
finally:
pool.join()
else:
# Run tests serially.
for command in self.cmd_list:
rc = run_command(command, verbosity=verbosity, out=out)
self.save_result(rc)
# Print ending summary.
t_end = time.time()
print >> self.out, "="*80
print >> self.out, ""
print >> self.out, "Tests finished. Elapsed time: %.2fs" %(t_end-t_start)
print >> self.out, ""
extra_stderr = 0
test_cases = []
# Process results for errors and warnings.
extra_stderr = len([result for result in self.results if result.stderr_lines])
longjobs = [result for result in self.results if result.wall_time > MAX_TIME]
warnings = [result for result in self.results if self.check_alert(result) == 1]
failures = [result for result in self.results if self.check_alert(result) == 2]
self.finished = len(self.results)
self.failure = len(failures)
self.warning = len(warnings)
# Output JUnit XML
if output_junit_xml:
from junit_xml import TestSuite, TestCase
for result in self.results:
tc = TestCase(name=result.command,
classname=result.command,
elapsed_sec=result.wall_time,
stdout='\n'.join(result.stdout_lines),
stderr='\n'.join(result.stderr_lines))
if result.return_code != 0:
tc.add_failure_info(message='exit code %d' %result.return_code)
#if len(result.stderr_lines):
#tc.add_error_info(output='\n'.join(result.stderr_lines))
test_cases.append(tc)
ts = TestSuite("libtbx.run_tests_parallel", test_cases=test_cases)
with open('output.xml', 'wb') as f:
print >> f, TestSuite.to_xml_string([ts], prettyprint=True)
# Run time distribution.
if (libtbx.env.has_module("scitbx")) :
from scitbx.array_family import flex
print >> self.out, "Distribution of test runtimes:"
hist = flex.histogram(flex.double([result.wall_time for result in self.results]), n_slots=10)
hist.show(f=self.out, prefix=" ", format_cutoffs="%.1fs")
print >> self.out, ""
# Long job warning.
if longjobs:
print >> self.out, ""
print >> self.out, "Warning: the following jobs took at least %d seconds:"%MAX_TIME
for result in sorted(longjobs, key=lambda result:result.wall_time):
print >> self.out, " %s: %.1fs"%(result.command, result.wall_time)
print >> self.out, "Please try to reduce overall runtime - consider splitting up these tests."
# Failures.
if failures:
print >> self.out, ""
print >> self.out, "Error: the following jobs returned non-zero exit codes or suspicious stderr output:"
print >> self.out, ""
for result in warnings:
self.display_result(result, alert=1, out=self.out, log_return=self.out, log_stderr=self.out)
for result in failures:
self.display_result(result, alert=2, out=self.out, log_return=self.out, log_stderr=self.out)
print >> self.out, ""
print >> self.out, "Please verify these tests manually."
print >> self.out, ""
# Summary
print >> self.out, "Summary:"
print >> self.out, " Tests run :",self.finished
print >> self.out, " Failures :",self.failure
print >> self.out, " Warnings (possible failures) :",self.warning
print >> self.out, " Stderr output (discouraged) :",extra_stderr
if (self.finished != len(self.cmd_list)) :
print >> self.out, "*" * 80
print >> self.out, " WARNING: NOT ALL TESTS FINISHED!"
print >> self.out, "*" * 80
def exporter_testcase_junit(test_result_ext, test_suite_properties=None):
"""! Export test results in JUnit XML compliant format
@param test_result_ext Extended report from Greentea
@param test_spec Dictionary of test build names to test suite properties
@details This function will import junit_xml library to perform report conversion
@return String containing Junit XML formatted test result output
"""
from junit_xml import TestSuite, TestCase
test_suites = []
for target_name in test_result_ext:
test_results = test_result_ext[target_name]
for test_suite_name in test_results:
test = test_results[test_suite_name]
# tc_elapsed_sec = test['elapsed_time']
tc_stdout = str() #test['single_test_output']
try:
tc_stdout = test['single_test_output'].decode('unicode_escape').encode('ascii','ignore')
except UnicodeDecodeError as e:
err_mgs = "(UnicodeDecodeError) exporter_testcase_junit:", str(e)
tc_stdout = err_mgs
print err_mgs
# testcase_result stores info about test case results
testcase_result = test['testcase_result']
# "testcase_result": {
# "STRINGS004": {
# "duration": 0.009999990463256836,
# "time_start": 1453073018.275,
# "time_end": 1453073018.285,
# "result": 1
# },
test_cases = []
for tc_name in sorted(testcase_result.keys()):
duration = testcase_result[tc_name].get('duration', 0.0)
utest_log = testcase_result[tc_name].get('utest_log', '')
result_text = testcase_result[tc_name].get('result_text', "UNDEF")
try:
tc_stderr = '\n'.join(utest_log).decode('unicode_escape').encode('ascii','ignore')
except UnicodeDecodeError as e:
err_mgs = "(UnicodeDecodeError) exporter_testcase_junit:" + str(e)
tc_stderr = err_mgs
print err_mgs
tc_class = target_name + '.' + test_suite_name
if result_text == 'SKIPPED':
# Skipped test cases do not have logs and we do not want to put
# whole log inside JUNIT for skipped test case
tc_stderr = str()
tc = TestCase(tc_name, tc_class, duration, tc_stdout, tc_stderr)
if result_text == 'FAIL':
tc.add_failure_info(result_text)
elif result_text == 'SKIPPED':
tc.add_skipped_info(result_text)
elif result_text != 'OK':
tc.add_error_info(result_text)
test_cases.append(tc)
ts_name = target_name
test_build_properties = test_suite_properties[target_name] if target_name in test_suite_properties else None
ts = TestSuite(ts_name, test_cases, properties=test_build_properties)
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
def get_junit_xml_string(n_passes=1, n_fails=1, n_skips=1, n_errors=1):
cases = _gen_cases(n_passes, n_fails, n_skips, n_errors)
suite = TestSuite("fake-junit-xml-suite", cases)
return TestSuite.to_xml_string([suite])
def tests_finished(self):
print(TestSuite.to_xml_string(self.test_suites))
def __init__ (self,
cmd_list,
nprocs=1,
out=sys.stdout,
log=None,
quiet=False,
output_junit_xml=False) :
if (log is None) : log = null_out()
self.out = multi_out()
self.log = log
self.out.register("stdout", out)
self.out.register("log", log)
self.quiet = quiet
self.cmd_list = []
for cmd in cmd_list :
if (not cmd in self.cmd_list) :
self.cmd_list.append(cmd)
else :
print >> self.out, " test %s repeated, skipping" % cmd
nprocs = min(nprocs, len(self.cmd_list))
print >> self.out, "\n Starting command list"
print >> self.out, " NProcs :",nprocs
print >> self.out, " Cmds :",len(self.cmd_list)
t_start = time.time()
if nprocs>1:
pool = Pool(processes=nprocs)
self.results = []
for command in self.cmd_list:
if nprocs>1:
pool.apply_async(
run_command,
[command, (not quiet), out],
callback=self.save_result)
else:
rc = run_command(command, verbose=(not quiet), out=out)
self.save_result(rc)
if nprocs>1:
try :
try :
pool.close()
except KeyboardInterrupt :
print >> self.out, "Caught KeyboardInterrupt, terminating"
pool.terminate()
finally :
pool.join()
print >> self.out, '\nProcesses have joined : %d\n' % len(self.results)
t_end = time.time()
print >> self.out, ""
print >> self.out, "Elapsed time: %.2fs" %(t_end-t_start)
print >> self.out, ""
finished = 0
warning = 0
extra_stderr = 0
failure = 0
failures = []
long_jobs = []
long_runtimes = []
runtimes = []
if output_junit_xml:
from junit_xml import TestSuite, TestCase
test_cases = []
for result in self.results :
finished += 1
runtimes.append(result.wall_time)
if (result.return_code != 0) :
failure += 1
failures.append(result)
else :
if (len(result.error_lines) != 0) :
warning += 1
failures.append(result)
if (len(result.stderr_lines) != 0):
extra_stderr += 1
if (result.wall_time > max_time) :
long_jobs.append(result.command)
long_runtimes.append(result.wall_time)
if output_junit_xml:
tc = TestCase(name=result.command,
classname=result.command,
elapsed_sec=result.wall_time,
stdout='\n'.join(result.stdout_lines),
stderr='\n'.join(result.stderr_lines))
if result.return_code != 0:
tc.add_failure_info(message='exit code %d' %result.return_code)
#if len(result.stderr_lines):
#tc.add_error_info(output='\n'.join(result.stderr_lines))
test_cases.append(tc)
if output_junit_xml:
ts = TestSuite("libtbx.run_tests_parallel", test_cases=test_cases)
with open('output.xml', 'wb') as f:
print >> f, TestSuite.to_xml_string([ts], prettyprint=True)
if (libtbx.env.has_module("scitbx")) :
from scitbx.array_family import flex
print >> self.out, "Distribution of test runtimes:"
hist = flex.histogram(flex.double(runtimes), n_slots=20)
hist.show(f=self.out, prefix=" ", format_cutoffs="%.1fs")
print >> self.out, ""
if (len(long_jobs) > 0) :
print >> self.out, ""
print >> self.out, "WARNING: the following jobs took at least %d seconds:" % \
max_time
jobs_and_timings = list(zip(long_jobs, long_runtimes))
jobs_and_timings.sort(lambda x,y: cmp(x[1], y[1]))
for cmd, runtime in jobs_and_timings :
print >> self.out, " " + cmd + " : %.1fs" % runtime
print >> self.out, "Please try to reduce overall runtime - consider splitting up these tests."
if (len(failures) > 0) :
print >> self.out, ""
print >> self.out, "ERROR: the following jobs returned non-zero exit codes or suspicious stderr output:"
for result in failures :
print >> self.out, ""
print >> self.out, result.command + "(exit code %d):" % result.return_code
for line in result.stderr_lines :
print >> self.out, " " + line
for line in result.error_lines :
print >> self.out, " " + line
print >> self.out, ""
print >> self.out, "Please verify these tests manually."
print >> self.out, ""
print >> self.out, "Summary:"
print >> self.out, " Tests run :",finished
print >> self.out, " Failures :",failure
print >> self.out, " Warnings (possible failures) :",warning
print >> self.out, " Stderr output (discouraged) :",extra_stderr
if (finished != len(self.cmd_list)) :
print >> self.out, "*" * 80
print >> self.out, " WARNING: NOT ALL TESTS FINISHED!"
print >> self.out, "*" * 80
def exporter_testcase_junit(test_result_ext, test_suite_properties=None):
"""! Export test results in JUnit XML compliant format
@param test_result_ext Extended report from Greentea
@param test_suite_properties Data from yotta module.json file
@details This function will import junit_xml library to perform report conversion
@return String containing Junit XML formatted test result output
"""
from junit_xml import TestSuite, TestCase
ym_name = test_suite_properties.get('name', 'unknown')
test_suites = []
for target_name in test_result_ext:
test_results = test_result_ext[target_name]
for test_suite_name in test_results:
test = test_results[test_suite_name]
# tc_elapsed_sec = test['elapsed_time']
tc_stdout = '' #test['single_test_output']
try:
tc_stderr = test['single_test_output'].decode('unicode_escape').encode('ascii','ignore')
except UnicodeDecodeError as e:
print "exporter_testcase_junit:", str(e)
# testcase_result stores info about test case results
testcase_result = test['testcase_result']
# "testcase_result": {
# "STRINGS004": {
# "duration": 0.009999990463256836,
# "time_start": 1453073018.275,
# "time_end": 1453073018.285,
# "result": 1
# },
test_cases = []
for tc_name in sorted(testcase_result.keys()):
duration = testcase_result[tc_name].get('duration', 0.0)
# result = testcase_result[tc_name].get('result', 0)
# passed = testcase_result[tc_name].get('passed', 0)
# failed = testcase_result[tc_name].get('failed', 0)
utest_log = testcase_result[tc_name].get('utest_log', '')
result_text = testcase_result[tc_name].get('result_text', "UNDEF")
try:
tc_stdout = '\n'.join(utest_log).decode('unicode_escape').encode('ascii','ignore')
except UnicodeDecodeError as e:
print "exporter_testcase_junit:", str(e)
tc_class = ym_name + '.' + target_name + '.' + test_suite_name
tc = TestCase(tc_name, tc_class, duration, tc_stdout, tc_stderr)
message = ''
if result_text == 'FAIL':
tc.add_failure_info(message, tc_stdout)
elif result_text != 'OK':
tc.add_error_info(message, tc_stdout)
test_cases.append(tc)
ts_name = ym_name + '.' + target_name
ts = TestSuite(ts_name, test_cases)
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
test_name = os.path.splitext(test_file)[0]
expected_output = test_directory + os.sep + test_name + ".spec"
if not os.path.isfile(expected_output):
raise Exception("Could not find expected_output file for test %r" % test_name)
# Actually run the test
code, sout, serr = run_with_timeout([executable, test_directory + os.sep + test_file], os.getcwd(), timeout=10)
if code != 0:
failed = True
log_errord(test_name, serr, sout)
else:
with open(expected_output, "r") as f:
expected = f.read()
if sout != expected:
failed = True
log_failed(test_name, sout, expected)
else:
log_passed(test_name)
if xml_out:
test_suite = TestSuite("Functional Tests", test_cases)
print TestSuite.to_xml_string([test_suite])
if failed and not xml_out:
sys.exit(1)
else:
sys.exit(0)
