def report(self):
"""Give a summary about testing results and
print detailed LilyPond log for any failed tests."""
print "Summary:\n"
print " {} failed tests out of {}".format(len(self.failed_tests),
len(self.test_files)), "\n"
if len(self.failed_tests) > 0:
fail_list = [t for t in self.failed_tests]
fail_list.sort()
print "Failed tests:"
for test in fail_list:
print "-", self.__relative_path(test)
print "\nDetails for failed tests:\n"
for test in fail_list:
test_name = self.__relative_path(test)
print test_name
print "-" * len(test_name)
print self.failed_tests[test]
print ""
print_separator()
return 1
return 0
def collect_tests(self):
"""Iterate over the whole openLilyLib directory and
collect valid test files, either all LilyPond files in
'usage-examples' directories or files specified in
.simple-tests-include files anywhere.
.simple-tests-excludes are used to skip files."""
print_separator()
print "Collecting test files\n"
# iterate over directory structure
for root, _, files in os.walk(self.openlilylib_dir):
self.__collect_all_in_dir(root)
# build definitive list of test cases:
self.test_files = [
t for t in self.test_files if t not in self.excluded_tests
]
self.test_files.extend(
[t for t in self.included_tests if t not in self.excluded_tests])
self.test_files = [
t for t in self.test_files if self.is_runnable_file(t)
]
self.test_files.sort()
# print summary about test cases
print "Found {} test files:".format(len(self.test_files))
print "\n".join([self.__relative_path(t) for t in self.test_files])
print "\n"
print "Potential test files explicitly excluded:"
print "\n".join([self.__relative_path(t) for t in self.excluded_tests])
def run(self):
"""Run the tests collected earlier"""
print_separator()
print "Running tests now\n"
for test in self.test_files:
print "Running test", self.__relative_path(test)
test_result_dir = os.path.join(self.openlilylib_dir,
"test",
"results",
self.lilypond_version,
os.path.dirname(self.__relative_path(test)))
if not os.path.exists(test_result_dir):
os.makedirs(test_result_dir)
returncode, out, err = self.lily_command.execute(
self.lily_command_with_includes_args + ['-o',
test_result_dir,
test])
if returncode == 0:
print "------- OK! --------"
else:
# if test failed, add it to the list of failed tests to be reported later
self.failed_tests[test] = err
print "\n====== FAILED ======"
print "See details at the end of test run."
print_separator()
def report(self):
"""Give a summary about testing results and
print detailed LilyPond log for any failed tests."""
print "Summary:\n"
print " {} failed tests out of {}".format(
len(self.failed_tests), len(self.test_files)), "\n"
if len(self.failed_tests) > 0:
fail_list = [t for t in self.failed_tests]
fail_list.sort()
print "Failed tests:"
for test in fail_list:
print "-", self.__relative_path(test)
print "\nDetails for failed tests:\n"
for test in fail_list:
test_name = self.__relative_path(test)
print test_name
print "-" * len(test_name)
print self.failed_tests[test]
print ""
print_separator()
return 1
return 0
def collect_tests(self):
"""Iterate over the whole openLilyLib directory and
collect valid test files, either all LilyPond files in
'usage-examples' directories or files specified in
.simple-tests-include files anywhere.
.simple-tests-excludes are used to skip files."""
print_separator()
print "Collecting test files\n"
# iterate over directory structure
for root, _, files in os.walk(self.openlilylib_dir):
self.__collect_all_in_dir(root)
# build definitive list of test cases:
self.test_files = [t for t in self.test_files
if t not in self.excluded_tests]
self.test_files.extend([t for t in self.included_tests
if t not in self.excluded_tests])
self.test_files = [t for t in self.test_files
if self.is_runnable_file(t)]
self.test_files.sort()
# print summary about test cases
print "Found {} test files:".format(len(self.test_files))
print "\n".join([self.__relative_path(t) for t in self.test_files])
print "\n"
print "Potential test files explicitly excluded:"
print "\n".join([self.__relative_path(t) for t in self.excluded_tests])
def run(self):
"""Run the tests collected earlier"""
print_separator()
print "Running tests now\n"
for test in self.test_files:
print "Running test", self.__relative_path(test)
test_result_dir = os.path.join(
self.openlilylib_dir, "test", "results", self.lilypond_version,
os.path.dirname(self.__relative_path(test)))
if not os.path.exists(test_result_dir):
os.makedirs(test_result_dir)
returncode, out, err = self.lily_command.execute(
self.lily_command_with_includes_args +
['-o', test_result_dir, test])
if returncode == 0:
print "------- OK! --------"
else:
# if test failed, add it to the list of failed tests to be reported later
self.failed_tests[test] = err
print "\n====== FAILED ======"
print "See details at the end of test run."
print_separator()
import os
import sys
from lilycmd import LilyCmd
from common_functions import print_separator
#############################################################
# Load environment variables
# at the same time checking if we're running on the CI server
try:
is_ci = os.environ["CI"]
lily_platform = os.environ["LILY_PLATFORM"]
lily_version = os.environ["LILY_VERSION"]
except:
sys.exit('\nScript can only be run in CI mode. Aborting\n')
#########################
# Actual script execution
if __name__ == "__main__":
print_separator()
print "============================="
print "openLilyLib automated testing"
print "============================="
print "Step 1:"
print "check LilyPond installation."
print "Requested LilyPond version: {}".format(lily_version)
LilyCmd.install(lily_platform, lily_version)
