def report(self): """Give a summary about testing results and print detailed LilyPond log for any failed tests.""" print "Summary:\n" print " {} failed tests out of {}".format(len(self.failed_tests), len(self.test_files)), "\n" if len(self.failed_tests) > 0: fail_list = [t for t in self.failed_tests] fail_list.sort() print "Failed tests:" for test in fail_list: print "-", self.__relative_path(test) print "\nDetails for failed tests:\n" for test in fail_list: test_name = self.__relative_path(test) print test_name print "-" * len(test_name) print self.failed_tests[test] print "" print_separator() return 1 return 0
def collect_tests(self): """Iterate over the whole openLilyLib directory and collect valid test files, either all LilyPond files in 'usage-examples' directories or files specified in .simple-tests-include files anywhere. .simple-tests-excludes are used to skip files.""" print_separator() print "Collecting test files\n" # iterate over directory structure for root, _, files in os.walk(self.openlilylib_dir): self.__collect_all_in_dir(root) # build definitive list of test cases: self.test_files = [ t for t in self.test_files if t not in self.excluded_tests ] self.test_files.extend( [t for t in self.included_tests if t not in self.excluded_tests]) self.test_files = [ t for t in self.test_files if self.is_runnable_file(t) ] self.test_files.sort() # print summary about test cases print "Found {} test files:".format(len(self.test_files)) print "\n".join([self.__relative_path(t) for t in self.test_files]) print "\n" print "Potential test files explicitly excluded:" print "\n".join([self.__relative_path(t) for t in self.excluded_tests])
def run(self): """Run the tests collected earlier""" print_separator() print "Running tests now\n" for test in self.test_files: print "Running test", self.__relative_path(test) test_result_dir = os.path.join(self.openlilylib_dir, "test", "results", self.lilypond_version, os.path.dirname(self.__relative_path(test))) if not os.path.exists(test_result_dir): os.makedirs(test_result_dir) returncode, out, err = self.lily_command.execute( self.lily_command_with_includes_args + ['-o', test_result_dir, test]) if returncode == 0: print "------- OK! --------" else: # if test failed, add it to the list of failed tests to be reported later self.failed_tests[test] = err print "\n====== FAILED ======" print "See details at the end of test run." print_separator()
def report(self): """Give a summary about testing results and print detailed LilyPond log for any failed tests.""" print "Summary:\n" print " {} failed tests out of {}".format( len(self.failed_tests), len(self.test_files)), "\n" if len(self.failed_tests) > 0: fail_list = [t for t in self.failed_tests] fail_list.sort() print "Failed tests:" for test in fail_list: print "-", self.__relative_path(test) print "\nDetails for failed tests:\n" for test in fail_list: test_name = self.__relative_path(test) print test_name print "-" * len(test_name) print self.failed_tests[test] print "" print_separator() return 1 return 0
def collect_tests(self): """Iterate over the whole openLilyLib directory and collect valid test files, either all LilyPond files in 'usage-examples' directories or files specified in .simple-tests-include files anywhere. .simple-tests-excludes are used to skip files.""" print_separator() print "Collecting test files\n" # iterate over directory structure for root, _, files in os.walk(self.openlilylib_dir): self.__collect_all_in_dir(root) # build definitive list of test cases: self.test_files = [t for t in self.test_files if t not in self.excluded_tests] self.test_files.extend([t for t in self.included_tests if t not in self.excluded_tests]) self.test_files = [t for t in self.test_files if self.is_runnable_file(t)] self.test_files.sort() # print summary about test cases print "Found {} test files:".format(len(self.test_files)) print "\n".join([self.__relative_path(t) for t in self.test_files]) print "\n" print "Potential test files explicitly excluded:" print "\n".join([self.__relative_path(t) for t in self.excluded_tests])
def run(self): """Run the tests collected earlier""" print_separator() print "Running tests now\n" for test in self.test_files: print "Running test", self.__relative_path(test) test_result_dir = os.path.join( self.openlilylib_dir, "test", "results", self.lilypond_version, os.path.dirname(self.__relative_path(test))) if not os.path.exists(test_result_dir): os.makedirs(test_result_dir) returncode, out, err = self.lily_command.execute( self.lily_command_with_includes_args + ['-o', test_result_dir, test]) if returncode == 0: print "------- OK! --------" else: # if test failed, add it to the list of failed tests to be reported later self.failed_tests[test] = err print "\n====== FAILED ======" print "See details at the end of test run." print_separator()
import os import sys from lilycmd import LilyCmd from common_functions import print_separator ############################################################# # Load environment variables # at the same time checking if we're running on the CI server try: is_ci = os.environ["CI"] lily_platform = os.environ["LILY_PLATFORM"] lily_version = os.environ["LILY_VERSION"] except: sys.exit('\nScript can only be run in CI mode. Aborting\n') ######################### # Actual script execution if __name__ == "__main__": print_separator() print "=============================" print "openLilyLib automated testing" print "=============================" print "Step 1:" print "check LilyPond installation." print "Requested LilyPond version: {}".format(lily_version) LilyCmd.install(lily_platform, lily_version)