def test_testsuite(self): '''(TestSuite) internal test suite''' build_dir = testlib.prepare_source('libjpeg6b', \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), self.patch_system) os.chdir(build_dir) package_version = testlib.get_changelog_version(build_dir) print "" print "Package version is %s" % package_version print " clean" rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'clean']) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) print " build (this will take a while)" rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'build']) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) print " tests (this will take considerably longer)" rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'make', '-C', build_dir, 'test']) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report)
def test_aa_build_source(self): '''Build nss''' if os.path.exists(self.cached_build_topdir): print "" build_dir = self._get_cached_build_dir() print " using cached build: %s" % build_dir return build_dir = testlib.prepare_source('nss', \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), self.patch_system) os.chdir(build_dir) print "" print " debian/rules clean" rc, report = testlib.cmd([ 'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'clean' ]) print " debian/rules build (this may take a while)" rc, report = testlib.cmd([ 'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'build' ]) self.cached_build_dir = os.path.join(self.cached_build_topdir, os.path.basename(build_dir)) os.makedirs(self.cached_build_topdir) shutil.move(build_dir, self.cached_build_dir) os.chdir(self.topdir)
def test_dbusglib_aa_tests(self): '''Test dbus-glib \'make check\' with --enable-tests''' build_dir = testlib.prepare_source('dbus-glib', \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), \ self.patch_system) os.chdir(build_dir) print "" print " make clean" rc, report = testlib.cmd( ['sudo', '-u', self.builder.login, 'make', 'clean']) print " autoreconf" rc, report = testlib.cmd( ['sudo', '-u', self.builder.login, 'autoreconf', '-f', '-i']) print " configure" rc, report = testlib.cmd([ 'sudo', '-u', self.builder.login, './configure', '--prefix=%s' % self.tmpdir, '--enable-tests', '--enable-asserts' ]) print " make (will take a while)" rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'make']) print " make check (will take a while)" rc, report = testlib.cmd( ['sudo', '-H', '-u', self.builder.login, 'make', 'check']) expected = 0 # This actually fails on Precise, Trusty and Utopic if self.lsb_release['Release'] in [12.04, 14.04, 14.10, 15.04]: expected = 2 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) # Useful to see if failure #print report search_terms = [ 'PASS: dbus-glib-test', 'PASS: test-compile-nested.sh', 'PASS: run-peer-test.sh' ] if self.lsb_release['Release'] >= 14.10: search_terms += [ 'PASS: test-errors', 'PASS: test-specialized-types' ] # This actually fails on Precise, Trusty and Utopic if self.lsb_release['Release'] in [12.04, 14.04, 14.10, 15.04]: search_terms.append('FAIL: run-test.sh') else: search_terms.append('PASS: run-test.sh') for search in search_terms: self.assertTrue(search in report, "Could not find '%s' in:\n%s" % (search, report))
def test_locales(self): '''Test installed locales (this will take a while)''' build_dir = testlib.prepare_source('ufw', \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), \ self.patch_system) locales_dir = "/usr/share/ufw/messages" self.assertTrue(os.path.exists(locales_dir), "Could not find '%s'" % locales_dir) pythons = ['python'] if self.lsb_release['Release'] >= 12.10: pythons.append("python3") exe = os.path.join(build_dir, "tests", "check-locales") if not os.path.exists(exe): exe = os.path.join(os.getcwd(), "ufw", "check-locales") self.assertTrue(os.path.exists(exe), "Could not find '%s'" % exe) locales = glob.glob("%s/*.mo" % locales_dir) locales.sort() print "" for python in pythons: print " %s:" % python lines = open(self.ufw_exe).readlines() lines[0] = '#!/usr/bin/%s\n' % python contents = "".join(lines) testlib.config_replace(self.ufw_exe, contents) for i in ['C'] + locales: loc = os.path.basename(i).split('.')[0] print " %s:" % loc, sys.stdout.flush() rc, report = testlib.cmd( [exe, '-f', '-i', '-d', locales_dir, '-l', loc]) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) self.assertFalse("FAIL" in report, "Found 'FAIL' in report:\n%s" % report) print "ok"
def test_zz_autopkgtests(self): '''Run autopkgtests''' if self.kernel_version.endswith('-goldfish'): return self._skipped( 'An apparmor dependency (initramfs-tools) cannot be met on goldfish' ) build_dir = testlib.prepare_source('click-apparmor', \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), \ self.patch_system) os.chdir(build_dir) # Now run the autopackage tests tests = glob.glob("./debian/tests/*") for f in tests: if os.path.isdir(f): continue elif os.path.basename(f) == "control": continue elif os.path.basename(f) == "run_testsuite-as-root": # we already did this above continue print(" %s" % os.path.basename(f)) adttmp = os.path.join(self.tmpdir, os.path.basename(f)) os.mkdir(adttmp) os.environ['ADTTMP'] = adttmp os.chmod(f, 0755) (rc, report) = testlib.cmd(f) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) self.assertFalse('\nFAIL' in report, "Found '\\nFAIL' in:\n%s" % (report))
def test_testsuite(self): '''Run testsuite as non-root and root''' if self.kernel_version.endswith('-goldfish'): return self._skipped( 'An apparmor dependency (initramfs-tools) cannot be met on goldfish' ) build_dir = testlib.prepare_source('click-apparmor', \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), \ self.patch_system) os.chdir(build_dir) # First as non-root for user in [self.builder.login, 'root']: (rc, report) = testlib.cmd(['sudo', '-u', user, './test-clicktool.py']) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) self.assertFalse('FAIL' in report, "Found 'FAIL' in:\n%s" % (report))
def _test_zz_testsuite(self): '''Test testsuite''' if self.lsb_release['Release'] < 11.04: return self._skipped("testsuite doesn't work on this release") self.cached_src = os.path.join(self.topdir, "source") self.patch_system = None self.builder = testlib.TestUser( ) #group='users',uidmin=2000,lower=True) testlib.cmd(['chgrp', self.builder.login, self.tmpdir]) os.chmod(self.tmpdir, 0775) build_dir = testlib.prepare_source('apt', \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), \ self.patch_system) os.chdir(build_dir) print "" # print " clean" # rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'clean']) # expected = 0 # result = 'Got exit code %d, expected %d\n' % (rc, expected) # self.assertEquals(expected, rc, result + report) print " build" rc, report = testlib.cmd([ 'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'build' ]) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) # Only Precise and higher have a usable testsuite print " tests", if self.lsb_release['Release'] < 12.04: print "(skipped in this release)" else: print "" rc, report = testlib.cmd( ['sudo', '-u', self.builder.login, 'make', 'test']) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) failure_txt = "" for line in report.splitlines(): if "FAIL" in line: failure_txt += line + "\n" self.assertTrue( failure_txt == "", "Found failures in report:\n%s\nLines with failures:\n%s" % (report, failure_txt)) # Only Natty and higher have integration tests print " integration tests", if self.lsb_release['Release'] < 11.04: print "(skipped in this release)" else: print "" testlib.cmd([ 'sudo', '-H', '-u', self.builder.login, 'mkdir', '-m', '0700', os.path.join("/home", self.builder.login, ".gnupg") ]) if self.lsb_release['Release'] >= 12.04: rc, report = testlib.cmd([ 'sudo', '-H', '-u', self.builder.login, 'make', '-C', 'test/integration', 'test' ]) else: os.chdir(os.path.join(build_dir, 'test', 'integration')) rc, report = testlib.cmd( ['sudo', '-H', '-u', self.builder.login, './run-tests']) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) # Attempt to parse test output of the form (as seen by pyunit, not # terminal output): # Testcase test-apt-get-autoremove: P P P P P P P P P P P # Testcase test-apt-get-changelog: P P P P P # -E: changelog download failed # +E: changelog for this version is not (yet) available; try https://launchpad.net/ubuntu/+source/apt/+changelog # ###FAILED### # ... efails = ['test-apt-key-net-update:'] # LP: #1013639 efails = [] if self.lsb_release['Release'] >= 12.04: efails += [ 'test-apt-get-changelog:', 'test-bug-593360-modifiers-in-names:', 'test-bug-601016-description-translation:', 'test-bug-612099-multiarch-conflicts:', 'test-bug-64141-install-dependencies-for-on-hold:', 'test-conflicts-loop:', 'test-ubuntu-bug-859188-multiarch-reinstall:', ] elif self.lsb_release['Release'] == 11.10: efails += [ 'test-apt-get-changelog:', 'test-apt-get-download:', 'test-bug-407511-fail-invalid-default-release:', 'test-bug-470115-new-and-tighten-recommends:', 'test-bug-549968-install-depends-of-not-installed:', 'test-bug-590041-prefer-non-virtual-packages:', 'test-bug-601016-description-translation:', 'test-bug-612099-multiarch-conflicts:', 'test-bug-618288-multiarch-same-lockstep:', 'test-bug-632221-cross-dependency-satisfaction:', 'test-bug-64141-install-dependencies-for-on-hold:', 'test-compressed-indexes:', 'test-disappearing-packages:', 'test-handling-broken-orgroups:', 'test-release-candidate-switching:', 'test-ubuntu-bug-802901-multiarch-early-remove:', 'test-ubuntu-bug-806274-install-suggests:', 'test-ubuntu-bug-835625-multiarch-lockstep-installed-first:', ] elif self.lsb_release['Release'] == 11.04: efails += [ 'test-bug-590041-prefer-non-virtual-packages:', 'test-bug-595691-empty-and-broken-archive-files:', 'test-bug-601016-description-translation:', 'test-bug-64141-install-dependencies-for-on-hold:', 'test-compressed-indexes:', 'test-disappearing-packages:', 'test-pdiff-usage:', ] failure_txt = "" testcase_txt = "" in_testcase = False for line in report.splitlines(): testcase_txt += line + "\n" if line.startswith("Testcase"): in_testcase = True testcase_txt = line + "\n" continue elif in_testcase and '###FAILED###' in line: ex_fail = False for e in efails: if e in testcase_txt.splitlines()[0]: ex_fail = True break if not ex_fail: testcase_txt += line + "\n" failure_txt += testcase_txt + "\n" self.assertTrue( failure_txt == "", "Found failures in report:\n%s\nLines with failures:\n%s" % (report, failure_txt))
def test_dbus_tests(self): '''Test D-Bus \'make check\' with --enable-tests''' if self.lsb_release['Release'] == 15.10: # This test fails in the upstream release due to two different # bugs. Skip it in Ubuntu 15.10. # https://bugs.freedesktop.org/show_bug.cgi?id=91684 # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=796167 return self._skipped("Test fails in upstream release") # This test partially depends on X11 and non-X11 display servers may # experience failures. This is a simple (and hackish) test to check for # X11 support. x11_sock_dir = '/tmp/.X11-unix' x11_sock_exists = True if not os.path.exists(x11_sock_dir) or not os.listdir(x11_sock_dir): x11_sock_exists = False build_dir = testlib.prepare_source('dbus', \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), \ self.patch_system) os.chdir(build_dir) tests_user = self.builder.login if self.lsb_release['Release'] >= 12.04 and os.environ.has_key( "SUDO_USER"): # We need to do this as the user with the X session, otherwise on # 11.04 and higher the test-autolaunch script fails. Tried using # xhost +SI:localuser:self.builder.login, but that didn't do # enough. This works fine, so no big woo. tests_user = os.environ["SUDO_USER"] testlib.cmd(['chown', '-R', tests_user, self.tmpdir]) print "" print " make clean" rc, report = testlib.cmd(['sudo', '-u', tests_user, 'make', 'clean']) print " autoreconf" rc, report = testlib.cmd( ['sudo', '-u', tests_user, 'autoreconf', '-f', '-i']) print " configure" rc, report = testlib.cmd([ 'sudo', '-u', tests_user, './configure', '--prefix=%s' % self.tmpdir, '--enable-tests', '--enable-asserts', '--with-test-socket-dir=%s' % self.tmpdir ]) for d in 'var/run/dbus', 'share/dbus-1/services': rc, report = testlib.cmd([ 'sudo', '-u', tests_user, 'mkdir', '-p', '%s/%s' % (self.tmpdir, d) ]) print " make (will take a while)" rc, report = testlib.cmd(['sudo', '-u', tests_user, 'make']) print " make check (will take a while)" rc, report = testlib.cmd( ['sudo', '-H', '-u', tests_user, 'make', 'check']) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) # Useful to see if failure #print report if self.lsb_release['Release'] >= 16.04: self.assertTrue( '# FAIL: 0' in report, "Found test failures in test run ouput:\n%s" % report) self.assertTrue( '# ERROR: 0' in report, "Found test errors in test run ouput:\n%s" % report) else: self._search_for_dbus_test_terms()
def test_testsuite(self): '''Test testsuite''' # useful for testing (ie get shell after setUp()) #subprocess.call(['bash']) self.builder = testlib.TestUser( ) #group='users',uidmin=2000,lower=True) testlib.cmd(['chgrp', self.builder.login, self.tmpdir]) os.chmod(self.tmpdir, 0775) build_dir = testlib.prepare_source(app, \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), \ self.patch_system) os.chdir(build_dir) print "" print " clean" rc, report = testlib.cmd([ 'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'clean' ]) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) print " build" rc, report = testlib.cmd([ 'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'build' ]) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) print " tests" rc, report = testlib.cmd( ['sudo', '-u', self.builder.login, 'make', 'check']) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) # raptor has a few expected failures in the rdfa testsuite and a couple # other places efails = [ 'Checking 0094.xhtml FAILED', 'Checking 0101.xhtml FAILED', 'Checking 0102.xhtml FAILED', 'Checking 0103.xhtml FAILED', 'Checking bad-15.rdf FAILED - parsing succeeded but should have failed (NFC test)', 'Checking bad-16.rdf FAILED - parsing succeeded but should have failed (NFC test)', 'Checking bad-17.rdf FAILED - parsing succeeded but should have failed (NFC test)', '4 tests FAILED: 0094.xhtml 0101.xhtml 0102.xhtml 0103.xhtml', ] failure_txt = "" for line in report.splitlines(): if line in efails: continue if "FAIL" in line: failure_txt += line + "\n" self.assertTrue( failure_txt == "", "Found failures in report:\n%s\nLines with failures (besides '%s'):\n%s" % (report, ",".join(efails), failure_txt))
def _run_test(self, test, runas, search_terms, python="python"): '''Run specific testsuite test''' build_dir = testlib.prepare_source('ufw', \ self.builder, \ self.cached_src, \ os.path.join(self.tmpdir, \ os.path.basename(self.cached_src)), \ self.patch_system) os.chdir(build_dir) user_args = [] if runas == 'non-root': user_args = ['sudo', '-u', self.builder.login] test_args = [] if test == "non-root": test_args = ['./run_tests.sh', '-s', '-i', python] elif test == "root": test_args = ['./run_tests.sh', '-s', '-i', python, 'root'] elif test == 'syntax-check': rc, report = testlib.cmd(['grep', test, './Makefile']) if rc == 0: test_args = ['make', test] elif test == 'man-check': rc, report = testlib.cmd(['grep', test, './Makefile']) if rc == 0: test_args = ['make', test] if len(test_args) == 0: return self._skipped("Skipped: TODO") print "" print " make clean" rc, report = testlib.cmd(user_args + ['make', 'clean']) print " %s (may take a while)" % (" ".join(test_args)) rc, report = testlib.cmd(user_args + test_args) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) # Useful to see if failure #print report for search in search_terms: self.assertTrue(search in report, "Could not find '%s' in:\n%s" % (search, report)) self.assertFalse('FAIL' in report, "Found 'FAIL' in:\n%s" % (report)) # Newer versions of ufw added capabilities tests. Lets make sure the # test chain is not present if test == "root": for exe in ['iptables', 'ip6tables']: rc, report = testlib.cmd([exe, '-L', '-n']) expected = 0 result = 'Got exit code %d, expected %d\n' % (rc, expected) self.assertEquals(expected, rc, result + report) search = 'caps-test' self.assertFalse(search in report, "Found '%s' in:\n%s" % (search, report)) return report