def test_testsuite(self):
'''(TestSuite) internal test suite'''
build_dir = testlib.prepare_source('libjpeg6b', \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)),
self.patch_system)
os.chdir(build_dir)
package_version = testlib.get_changelog_version(build_dir)
print ""
print "Package version is %s" % package_version
print " clean"
rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'clean'])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
print " build (this will take a while)"
rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'build'])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
print " tests (this will take considerably longer)"
rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'make', '-C', build_dir, 'test'])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
def test_aa_build_source(self):
'''Build nss'''
if os.path.exists(self.cached_build_topdir):
print ""
build_dir = self._get_cached_build_dir()
print " using cached build: %s" % build_dir
return
build_dir = testlib.prepare_source('nss', \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)),
self.patch_system)
os.chdir(build_dir)
print ""
print " debian/rules clean"
rc, report = testlib.cmd([
'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules',
'clean'
])
print " debian/rules build (this may take a while)"
rc, report = testlib.cmd([
'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules',
'build'
])
self.cached_build_dir = os.path.join(self.cached_build_topdir,
os.path.basename(build_dir))
os.makedirs(self.cached_build_topdir)
shutil.move(build_dir, self.cached_build_dir)
os.chdir(self.topdir)
def test_dbusglib_aa_tests(self):
'''Test dbus-glib \'make check\' with --enable-tests'''
build_dir = testlib.prepare_source('dbus-glib', \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)), \
self.patch_system)
os.chdir(build_dir)
print ""
print " make clean"
rc, report = testlib.cmd(
['sudo', '-u', self.builder.login, 'make', 'clean'])
print " autoreconf"
rc, report = testlib.cmd(
['sudo', '-u', self.builder.login, 'autoreconf', '-f', '-i'])
print " configure"
rc, report = testlib.cmd([
'sudo', '-u', self.builder.login, './configure',
'--prefix=%s' % self.tmpdir, '--enable-tests', '--enable-asserts'
])
print " make (will take a while)"
rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'make'])
print " make check (will take a while)"
rc, report = testlib.cmd(
['sudo', '-H', '-u', self.builder.login, 'make', 'check'])
expected = 0
# This actually fails on Precise, Trusty and Utopic
if self.lsb_release['Release'] in [12.04, 14.04, 14.10, 15.04]:
expected = 2
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
# Useful to see if failure
#print report
search_terms = [
'PASS: dbus-glib-test', 'PASS: test-compile-nested.sh',
'PASS: run-peer-test.sh'
]
if self.lsb_release['Release'] >= 14.10:
search_terms += [
'PASS: test-errors', 'PASS: test-specialized-types'
]
# This actually fails on Precise, Trusty and Utopic
if self.lsb_release['Release'] in [12.04, 14.04, 14.10, 15.04]:
search_terms.append('FAIL: run-test.sh')
else:
search_terms.append('PASS: run-test.sh')
for search in search_terms:
self.assertTrue(search in report,
"Could not find '%s' in:\n%s" % (search, report))
def test_locales(self):
'''Test installed locales (this will take a while)'''
build_dir = testlib.prepare_source('ufw', \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)), \
self.patch_system)
locales_dir = "/usr/share/ufw/messages"
self.assertTrue(os.path.exists(locales_dir),
"Could not find '%s'" % locales_dir)
pythons = ['python']
if self.lsb_release['Release'] >= 12.10:
pythons.append("python3")
exe = os.path.join(build_dir, "tests", "check-locales")
if not os.path.exists(exe):
exe = os.path.join(os.getcwd(), "ufw", "check-locales")
self.assertTrue(os.path.exists(exe), "Could not find '%s'" % exe)
locales = glob.glob("%s/*.mo" % locales_dir)
locales.sort()
print ""
for python in pythons:
print " %s:" % python
lines = open(self.ufw_exe).readlines()
lines[0] = '#!/usr/bin/%s\n' % python
contents = "".join(lines)
testlib.config_replace(self.ufw_exe, contents)
for i in ['C'] + locales:
loc = os.path.basename(i).split('.')[0]
print " %s:" % loc,
sys.stdout.flush()
rc, report = testlib.cmd(
[exe, '-f', '-i', '-d', locales_dir, '-l', loc])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
self.assertFalse("FAIL" in report,
"Found 'FAIL' in report:\n%s" % report)
print "ok"
def test_zz_autopkgtests(self):
'''Run autopkgtests'''
if self.kernel_version.endswith('-goldfish'):
return self._skipped(
'An apparmor dependency (initramfs-tools) cannot be met on goldfish'
)
build_dir = testlib.prepare_source('click-apparmor', \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)), \
self.patch_system)
os.chdir(build_dir)
# Now run the autopackage tests
tests = glob.glob("./debian/tests/*")
for f in tests:
if os.path.isdir(f):
continue
elif os.path.basename(f) == "control":
continue
elif os.path.basename(f) == "run_testsuite-as-root":
# we already did this above
continue
print(" %s" % os.path.basename(f))
adttmp = os.path.join(self.tmpdir, os.path.basename(f))
os.mkdir(adttmp)
os.environ['ADTTMP'] = adttmp
os.chmod(f, 0755)
(rc, report) = testlib.cmd(f)
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
self.assertFalse('\nFAIL' in report,
"Found '\\nFAIL' in:\n%s" % (report))
def test_testsuite(self):
'''Run testsuite as non-root and root'''
if self.kernel_version.endswith('-goldfish'):
return self._skipped(
'An apparmor dependency (initramfs-tools) cannot be met on goldfish'
)
build_dir = testlib.prepare_source('click-apparmor', \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)), \
self.patch_system)
os.chdir(build_dir)
# First as non-root
for user in [self.builder.login, 'root']:
(rc,
report) = testlib.cmd(['sudo', '-u', user, './test-clicktool.py'])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
self.assertFalse('FAIL' in report,
"Found 'FAIL' in:\n%s" % (report))
def _test_zz_testsuite(self):
'''Test testsuite'''
if self.lsb_release['Release'] < 11.04:
return self._skipped("testsuite doesn't work on this release")
self.cached_src = os.path.join(self.topdir, "source")
self.patch_system = None
self.builder = testlib.TestUser(
) #group='users',uidmin=2000,lower=True)
testlib.cmd(['chgrp', self.builder.login, self.tmpdir])
os.chmod(self.tmpdir, 0775)
build_dir = testlib.prepare_source('apt', \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)), \
self.patch_system)
os.chdir(build_dir)
print ""
# print " clean"
# rc, report = testlib.cmd(['sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules', 'clean'])
# expected = 0
# result = 'Got exit code %d, expected %d\n' % (rc, expected)
# self.assertEquals(expected, rc, result + report)
print " build"
rc, report = testlib.cmd([
'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules',
'build'
])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
# Only Precise and higher have a usable testsuite
print " tests",
if self.lsb_release['Release'] < 12.04:
print "(skipped in this release)"
else:
print ""
rc, report = testlib.cmd(
['sudo', '-u', self.builder.login, 'make', 'test'])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
failure_txt = ""
for line in report.splitlines():
if "FAIL" in line:
failure_txt += line + "\n"
self.assertTrue(
failure_txt == "",
"Found failures in report:\n%s\nLines with failures:\n%s" %
(report, failure_txt))
# Only Natty and higher have integration tests
print " integration tests",
if self.lsb_release['Release'] < 11.04:
print "(skipped in this release)"
else:
print ""
testlib.cmd([
'sudo', '-H', '-u', self.builder.login, 'mkdir', '-m', '0700',
os.path.join("/home", self.builder.login, ".gnupg")
])
if self.lsb_release['Release'] >= 12.04:
rc, report = testlib.cmd([
'sudo', '-H', '-u', self.builder.login, 'make', '-C',
'test/integration', 'test'
])
else:
os.chdir(os.path.join(build_dir, 'test', 'integration'))
rc, report = testlib.cmd(
['sudo', '-H', '-u', self.builder.login, './run-tests'])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
# Attempt to parse test output of the form (as seen by pyunit, not
# terminal output):
# Testcase test-apt-get-autoremove: P P P P P P P P P P P
# Testcase test-apt-get-changelog: P P P P P
# -E: changelog download failed
# +E: changelog for this version is not (yet) available; try https://launchpad.net/ubuntu/+source/apt/+changelog
# ###FAILED###
# ...
efails = ['test-apt-key-net-update:'] # LP: #1013639
efails = []
if self.lsb_release['Release'] >= 12.04:
efails += [
'test-apt-get-changelog:',
'test-bug-593360-modifiers-in-names:',
'test-bug-601016-description-translation:',
'test-bug-612099-multiarch-conflicts:',
'test-bug-64141-install-dependencies-for-on-hold:',
'test-conflicts-loop:',
'test-ubuntu-bug-859188-multiarch-reinstall:',
]
elif self.lsb_release['Release'] == 11.10:
efails += [
'test-apt-get-changelog:',
'test-apt-get-download:',
'test-bug-407511-fail-invalid-default-release:',
'test-bug-470115-new-and-tighten-recommends:',
'test-bug-549968-install-depends-of-not-installed:',
'test-bug-590041-prefer-non-virtual-packages:',
'test-bug-601016-description-translation:',
'test-bug-612099-multiarch-conflicts:',
'test-bug-618288-multiarch-same-lockstep:',
'test-bug-632221-cross-dependency-satisfaction:',
'test-bug-64141-install-dependencies-for-on-hold:',
'test-compressed-indexes:',
'test-disappearing-packages:',
'test-handling-broken-orgroups:',
'test-release-candidate-switching:',
'test-ubuntu-bug-802901-multiarch-early-remove:',
'test-ubuntu-bug-806274-install-suggests:',
'test-ubuntu-bug-835625-multiarch-lockstep-installed-first:',
]
elif self.lsb_release['Release'] == 11.04:
efails += [
'test-bug-590041-prefer-non-virtual-packages:',
'test-bug-595691-empty-and-broken-archive-files:',
'test-bug-601016-description-translation:',
'test-bug-64141-install-dependencies-for-on-hold:',
'test-compressed-indexes:',
'test-disappearing-packages:',
'test-pdiff-usage:',
]
failure_txt = ""
testcase_txt = ""
in_testcase = False
for line in report.splitlines():
testcase_txt += line + "\n"
if line.startswith("Testcase"):
in_testcase = True
testcase_txt = line + "\n"
continue
elif in_testcase and '###FAILED###' in line:
ex_fail = False
for e in efails:
if e in testcase_txt.splitlines()[0]:
ex_fail = True
break
if not ex_fail:
testcase_txt += line + "\n"
failure_txt += testcase_txt + "\n"
self.assertTrue(
failure_txt == "",
"Found failures in report:\n%s\nLines with failures:\n%s" %
(report, failure_txt))
def test_dbus_tests(self):
'''Test D-Bus \'make check\' with --enable-tests'''
if self.lsb_release['Release'] == 15.10:
# This test fails in the upstream release due to two different
# bugs. Skip it in Ubuntu 15.10.
# https://bugs.freedesktop.org/show_bug.cgi?id=91684
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=796167
return self._skipped("Test fails in upstream release")
# This test partially depends on X11 and non-X11 display servers may
# experience failures. This is a simple (and hackish) test to check for
# X11 support.
x11_sock_dir = '/tmp/.X11-unix'
x11_sock_exists = True
if not os.path.exists(x11_sock_dir) or not os.listdir(x11_sock_dir):
x11_sock_exists = False
build_dir = testlib.prepare_source('dbus', \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)), \
self.patch_system)
os.chdir(build_dir)
tests_user = self.builder.login
if self.lsb_release['Release'] >= 12.04 and os.environ.has_key(
"SUDO_USER"):
# We need to do this as the user with the X session, otherwise on
# 11.04 and higher the test-autolaunch script fails. Tried using
# xhost +SI:localuser:self.builder.login, but that didn't do
# enough. This works fine, so no big woo.
tests_user = os.environ["SUDO_USER"]
testlib.cmd(['chown', '-R', tests_user, self.tmpdir])
print ""
print " make clean"
rc, report = testlib.cmd(['sudo', '-u', tests_user, 'make', 'clean'])
print " autoreconf"
rc, report = testlib.cmd(
['sudo', '-u', tests_user, 'autoreconf', '-f', '-i'])
print " configure"
rc, report = testlib.cmd([
'sudo', '-u', tests_user, './configure',
'--prefix=%s' % self.tmpdir, '--enable-tests', '--enable-asserts',
'--with-test-socket-dir=%s' % self.tmpdir
])
for d in 'var/run/dbus', 'share/dbus-1/services':
rc, report = testlib.cmd([
'sudo', '-u', tests_user, 'mkdir', '-p',
'%s/%s' % (self.tmpdir, d)
])
print " make (will take a while)"
rc, report = testlib.cmd(['sudo', '-u', tests_user, 'make'])
print " make check (will take a while)"
rc, report = testlib.cmd(
['sudo', '-H', '-u', tests_user, 'make', 'check'])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
# Useful to see if failure
#print report
if self.lsb_release['Release'] >= 16.04:
self.assertTrue(
'# FAIL: 0' in report,
"Found test failures in test run ouput:\n%s" % report)
self.assertTrue(
'# ERROR: 0' in report,
"Found test errors in test run ouput:\n%s" % report)
else:
self._search_for_dbus_test_terms()
def test_testsuite(self):
'''Test testsuite'''
# useful for testing (ie get shell after setUp())
#subprocess.call(['bash'])
self.builder = testlib.TestUser(
) #group='users',uidmin=2000,lower=True)
testlib.cmd(['chgrp', self.builder.login, self.tmpdir])
os.chmod(self.tmpdir, 0775)
build_dir = testlib.prepare_source(app, \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)), \
self.patch_system)
os.chdir(build_dir)
print ""
print " clean"
rc, report = testlib.cmd([
'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules',
'clean'
])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
print " build"
rc, report = testlib.cmd([
'sudo', '-u', self.builder.login, 'fakeroot', 'debian/rules',
'build'
])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
print " tests"
rc, report = testlib.cmd(
['sudo', '-u', self.builder.login, 'make', 'check'])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
# raptor has a few expected failures in the rdfa testsuite and a couple
# other places
efails = [
'Checking 0094.xhtml FAILED',
'Checking 0101.xhtml FAILED',
'Checking 0102.xhtml FAILED',
'Checking 0103.xhtml FAILED',
'Checking bad-15.rdf FAILED - parsing succeeded but should have failed (NFC test)',
'Checking bad-16.rdf FAILED - parsing succeeded but should have failed (NFC test)',
'Checking bad-17.rdf FAILED - parsing succeeded but should have failed (NFC test)',
'4 tests FAILED: 0094.xhtml 0101.xhtml 0102.xhtml 0103.xhtml',
]
failure_txt = ""
for line in report.splitlines():
if line in efails:
continue
if "FAIL" in line:
failure_txt += line + "\n"
self.assertTrue(
failure_txt == "",
"Found failures in report:\n%s\nLines with failures (besides '%s'):\n%s"
% (report, ",".join(efails), failure_txt))
def _run_test(self, test, runas, search_terms, python="python"):
'''Run specific testsuite test'''
build_dir = testlib.prepare_source('ufw', \
self.builder, \
self.cached_src, \
os.path.join(self.tmpdir, \
os.path.basename(self.cached_src)), \
self.patch_system)
os.chdir(build_dir)
user_args = []
if runas == 'non-root':
user_args = ['sudo', '-u', self.builder.login]
test_args = []
if test == "non-root":
test_args = ['./run_tests.sh', '-s', '-i', python]
elif test == "root":
test_args = ['./run_tests.sh', '-s', '-i', python, 'root']
elif test == 'syntax-check':
rc, report = testlib.cmd(['grep', test, './Makefile'])
if rc == 0:
test_args = ['make', test]
elif test == 'man-check':
rc, report = testlib.cmd(['grep', test, './Makefile'])
if rc == 0:
test_args = ['make', test]
if len(test_args) == 0:
return self._skipped("Skipped: TODO")
print ""
print " make clean"
rc, report = testlib.cmd(user_args + ['make', 'clean'])
print " %s (may take a while)" % (" ".join(test_args))
rc, report = testlib.cmd(user_args + test_args)
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
# Useful to see if failure
#print report
for search in search_terms:
self.assertTrue(search in report,
"Could not find '%s' in:\n%s" % (search, report))
self.assertFalse('FAIL' in report, "Found 'FAIL' in:\n%s" % (report))
# Newer versions of ufw added capabilities tests. Lets make sure the
# test chain is not present
if test == "root":
for exe in ['iptables', 'ip6tables']:
rc, report = testlib.cmd([exe, '-L', '-n'])
expected = 0
result = 'Got exit code %d, expected %d\n' % (rc, expected)
self.assertEquals(expected, rc, result + report)
search = 'caps-test'
self.assertFalse(search in report,
"Found '%s' in:\n%s" % (search, report))
return report