def save_system_state(logdir):
# save package state to be able to re-create failures
try:
from apt_clone import AptClone
except ImportError:
logging.error("failed to import AptClone")
return
target = os.path.join(logdir, "apt-clone_system_state.tar.gz")
logging.debug("creating statefile: '%s'" % target)
# this file may contain sensitive data so ensure we create with the
# right umask
old_umask = os.umask(0066)
clone = AptClone()
clone.save_state(sourcedir="/",
target=target,
with_dpkg_status=True,
scrub_sources=True)
# reset umask
os.umask(old_umask)
# lspci output
try:
s = subprocess.Popen(["lspci", "-nn"],
stdout=subprocess.PIPE).communicate()[0]
open(os.path.join(logdir, "lspci.txt"), "w").write(s)
except OSError, e:
logging.debug("lspci failed: %s" % e)
def test_restore_state_on_new_distro_release_livecd(self, mock_lowlevel):
"""
test lucid -> maverick apt-clone-ugprade as if it will be used
from a live cd based upgrader
"""
# setup mock for dpkg -i
mock_lowlevel.install_debs.return_value = True
# create target dir
targetdir = self.tempdir
# status file from maverick (to simulate running on a maverick live-cd)
with open("./data/dpkg-status/dpkg-status-ubuntu-maverick",
"rb") as fp:
s = fp.read().decode("utf8")
s = s.replace(
"Architecture: i386",
"Architecture: %s" % apt_pkg.config.find("Apt::Architecture"))
path = os.path.join(targetdir, "var/lib/dpkg", "status")
with open(path, "wb",) as fp:
fp.write(s.encode("utf-8"))
# test upgrade clone from lucid system to maverick
clone = AptClone(cache_cls=MockAptCache)
clone.restore_state(
"./data/apt-state-ubuntu-lucid.tar.gz",
targetdir,
new_distro="maverick")
sources_list = os.path.join(targetdir, "etc","apt","sources.list")
self.assertTrue(os.path.exists(sources_list))
with open(sources_list) as fp:
self.assertTrue("maverick" in fp.read())
with open(sources_list) as fp:
self.assertFalse("lucid" in fp.read())
def test_clone_upgrade_synthetic(self):
""" test clone upgrade with on-the-fly generated chroots """
supported = distro_info.UbuntuDistroInfo().supported()
for meta in [
"ubuntu-standard", "ubuntu-desktop", "kubuntu-desktop",
"xubuntu-desktop"
]:
logging.info("testing %s" % meta)
old = self._create_fake_upgradable_root(supported[-2], meta=meta)
self.addCleanup(shutil.rmtree, old)
# create statefile based on the old data
with tarfile.open("lala.tar.gz", "w:gz") as state:
state.add(os.path.join(old, "var", "lib", "apt-clone",
"installed.pkgs"),
arcname="./var/lib/apt-clone/installed.pkgs")
# create new fake environment and try to upgrade
new = self._create_fake_upgradable_root(supported[-1], meta=meta)
self.addCleanup(shutil.rmtree, new)
cache = apt.Cache(rootdir=new)
clone = AptClone()
clone._restore_package_selection_in_cache("lala.tar.gz",
cache,
protect_installed=True)
self.assertFalse(cache[meta].marked_delete,
"package %s marked for removal" % meta)
self.assertTrue(len(cache.get_changes()) > 0)
def test_restore_state_simulate_with_new_release(self):
#apt_pkg.config.set("Debug::PkgProblemResolver", "1")
apt_pkg.config.set("Dir::state::status",
"./data/dpkg-status/dpkg-status-ubuntu-maverick")
clone = AptClone()
missing = clone.simulate_restore_state(
"./data/apt-state-ubuntu-lucid.tar.gz", "maverick")
def test_restore_state_simulate(self):
clone = AptClone()
supported = distro_info.UbuntuDistroInfo().supported()
missing = clone.simulate_restore_state("./data/apt-state.tar.gz", new_distro=supported[-1], exclude_pkgs=[])
# missing, because clone does not have universe enabled
self.assertEqual(list(missing), ['accerciser'])
def test_restore_state_on_new_distro_release_livecd(self, mock_lowlevel):
"""
test lucid -> maverick apt-clone-ugprade as if it will be used
from a live cd based upgrader
"""
# setup mock for dpkg -i
mock_lowlevel.install_debs.return_value = True
# create target dir
targetdir = self.tempdir
# status file from maverick (to simulate running on a maverick live-cd)
with open("./data/dpkg-status/dpkg-status-ubuntu-maverick",
"rb") as fp:
s = fp.read().decode("utf8")
s = s.replace(
"Architecture: i386",
"Architecture: %s" % apt_pkg.config.find("Apt::Architecture"))
path = os.path.join(targetdir, "var/lib/dpkg", "status")
with open(
path,
"wb",
) as fp:
fp.write(s.encode("utf-8"))
# test upgrade clone from lucid system to maverick
clone = AptClone(cache_cls=MockAptCache)
clone.restore_state("./data/apt-state-ubuntu-lucid.tar.gz",
targetdir,
new_distro="maverick")
sources_list = os.path.join(targetdir, "etc", "apt", "sources.list")
self.assertTrue(os.path.exists(sources_list))
with open(sources_list) as fp:
self.assertTrue("maverick" in fp.read())
with open(sources_list) as fp:
self.assertFalse("lucid" in fp.read())
def _save_state(self, with_dpkg_repack):
# setup mock
targetdir = self.tempdir
# test
clone = AptClone(cache_cls=MockAptCache)
sourcedir = "./data/mock-system"
clone.save_state(sourcedir, targetdir, with_dpkg_repack, with_dpkg_status=True)
# verify that we got the tarfile
tarname = os.path.join(targetdir, clone.CLONE_FILENAME)
self.assertTrue(os.path.exists(tarname))
with tarfile.open(tarname) as tar:
#print(tar.getmembers())
# verify members in tar
members = [m.name for m in tar.getmembers()]
self.assertTrue("./etc/apt/sources.list" in members)
self.assertTrue("./var/lib/apt-clone/installed.pkgs" in members)
self.assertTrue("./var/lib/apt-clone/foreign.pkgs" in members)
self.assertTrue("./var/lib/apt-clone/extended_states" in members)
self.assertTrue("./var/lib/apt-clone/dpkg-status" in members)
self.assertTrue("./etc/apt/sources.list.d" in members)
self.assertTrue("./etc/apt/preferences.d" in members)
self.assertTrue("./etc/apt/preferences" in members)
if clone.not_downloadable:
self.assertEqual(clone.commands.repack_deb.called, with_dpkg_repack)
# ensure we have no duplicates in the sources.list.d
sources_list_d = [p for p in members
if p.startswith("./etc/apt/sources.list.d")]
self.assertEqual(
sorted(sources_list_d),
sorted(
['./etc/apt/sources.list.d',
'./etc/apt/sources.list.d/ubuntu-mozilla-daily-ppa-maverick.list']))
def test_restore_state_simulate_with_new_release(self):
#apt_pkg.config.set("Debug::PkgProblemResolver", "1")
apt_pkg.config.set(
"Dir::state::status",
"./data/dpkg-status/dpkg-status-ubuntu-maverick")
clone = AptClone()
missing = clone.simulate_restore_state(
"./data/apt-state-ubuntu-lucid.tar.gz", "maverick")
def test_clone_upgrade_regression(self):
""" regression test against known installs """
new = self._create_fake_upgradable_root("natty", meta="ubuntu-desktop")
self.addCleanup(shutil.rmtree, new)
cache = apt.Cache(rootdir=new)
clone = AptClone()
clone._restore_package_selection_in_cache(
"./data/regression/apt-clone-state-ubuntu.tar.gz", cache)
self.assertTrue(len(cache.get_changes()) > 0)
def test_clone_upgrade_regression(self):
""" regression test against known installs """
new = self._create_fake_upgradable_root("natty", meta="ubuntu-desktop")
self.addCleanup(shutil.rmtree, new)
cache = apt.Cache(rootdir=new)
clone = AptClone()
clone._restore_package_selection_in_cache(
"./data/regression/apt-clone-state-ubuntu.tar.gz", cache)
self.assertTrue(len(cache.get_changes()) > 0)
def test_restore_state_simulate(self):
clone = AptClone()
supported = distro_info.UbuntuDistroInfo().supported()
missing = clone.simulate_restore_state("./data/apt-state.tar.gz",
new_distro=supported[-1],
exclude_pkgs=[])
# missing, because clone does not have universe enabled
self.assertEqual(list(missing), ['accerciser'])
def test_restore_state(self, mock_lowlevel):
# setup mock
mock_lowlevel.install_debs.return_value = True
targetdir = self.tempdir
# test
clone = AptClone(cache_cls=MockAptCache)
clone.restore_state(
"./data/apt-state_chroot_with_vim.tar.gz", targetdir)
self.assertTrue(
os.path.exists(os.path.join(targetdir, "etc","apt","sources.list")))
def test_restore_state_with_not_downloadable_debs(self, mock_lowlevel):
# setup mock
mock_lowlevel.install_debs.return_value = True
targetdir = self.tempdir
# test
clone = AptClone(cache_cls=MockAptCache)
clone.restore_state(
"./data/apt-state_with_not_downloadable_debs.tar.gz", targetdir)
self.assertTrue(
os.path.exists(
os.path.join(targetdir, "var", "lib", "apt-clone", "debs", "foo.deb")))
def test_restore_state(self, mock_lowlevel):
# setup mock
mock_lowlevel.install_debs.return_value = True
targetdir = self.tempdir
# test
clone = AptClone(cache_cls=MockAptCache)
clone.restore_state("./data/apt-state_chroot_with_vim.tar.gz",
targetdir)
self.assertTrue(
os.path.exists(
os.path.join(targetdir, "etc", "apt", "sources.list")))
def test_restore_state_with_not_downloadable_debs(self, mock_lowlevel):
# setup mock
mock_lowlevel.install_debs.return_value = True
targetdir = self.tempdir
# test
clone = AptClone(cache_cls=MockAptCache)
clone.restore_state(
"./data/apt-state_with_not_downloadable_debs.tar.gz", targetdir)
self.assertTrue(
os.path.exists(
os.path.join(targetdir, "var", "lib", "apt-clone", "debs",
"foo.deb")))
def setUp(self):
self.apt_clone = AptClone()
self.test_sources_fname = "test-sources.list"
with open(self.test_sources_fname, "wb") as f:
f.write(u"""# äüö
deb http://mvo:[email protected]/ ubuntu main
""".encode("utf-8"))
def save_system_state(logdir):
# save package state to be able to re-create failures
try:
from apt_clone import AptClone
except ImportError:
logging.error("failed to import AptClone")
return
target = os.path.join(logdir, "apt-clone_system_state.tar.gz")
logging.debug("creating statefile: '%s'" % target)
clone = AptClone()
clone.save_state(sourcedir="/", target=target, with_dpkg_status=True)
# lspci output
try:
s=subprocess.Popen(["lspci","-nn"], stdout=subprocess.PIPE).communicate()[0]
open(os.path.join(logdir, "lspci.txt"), "w").write(s)
except OSError, e:
logging.debug("lspci failed: %s" % e)
def test_unowned_in_etc(self):
# test in mock environement
apt_pkg.config.set("Dir::state::status",
"./data/mock-system/var/lib/dpkg/status")
clone = AptClone()
unowned = clone._find_unowned_in_etc("./data/mock-system")
self.assertFalse("/etc/conffile.modified" in unowned)
self.assertFalse("/etc/conffile.not-modified" in unowned)
self.assertTrue("/etc/unowned-file" in unowned)
# test on the real system and do very light checks
apt_pkg.config.set("Dir::state::status", "/var/lib/dpkg/status")
unowned = clone._find_unowned_in_etc()
#print(unowned)
self.assertNotEqual(unowned, set())
# negative test, is created by the installer
self.assertTrue("/etc/apt/sources.list" in unowned)
# postivie test, belongs to base-files
self.assertFalse("/etc/issue" in unowned)
def test_real(self):
if os.getuid() != 0:
print("Skipping because uid != 0")
return
# do it
target = "./test-chroot"
if not os.path.exists(target):
os.mkdir(target)
subprocess.call(["debootstrap", "--arch=i386",
"maverick", target])
# force i386
with open(os.path.join(target, "etc/apt/apt.conf"), "w") as fp:
fp.write('APT::Architecture "i386";')
# restore
clone = AptClone()
clone.restore_state(
"./data/apt-state_chroot_with_vim.tar.gz", target, "maverick")
def test_unowned_in_etc(self):
# test in mock environement
apt_pkg.config.set(
"Dir::state::status",
"./data/mock-system/var/lib/dpkg/status")
clone = AptClone()
unowned = clone._find_unowned_in_etc("./data/mock-system")
self.assertFalse("/etc/conffile.modified" in unowned)
self.assertFalse("/etc/conffile.not-modified" in unowned)
self.assertTrue("/etc/unowned-file" in unowned)
# test on the real system and do very light checks
apt_pkg.config.set(
"Dir::state::status",
"/var/lib/dpkg/status")
unowned = clone._find_unowned_in_etc()
#print(unowned)
self.assertNotEqual(unowned, set())
# negative test, is created by the installer
self.assertTrue("/etc/apt/sources.list" in unowned)
# postivie test, belongs to base-files
self.assertFalse("/etc/issue" in unowned)
def test_merge_sources(self):
clone = AptClone()
tmpdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmpdir)
sources_list = os.path.join(tmpdir, "etc", "apt", "sources.list")
os.makedirs(os.path.dirname(sources_list))
shutil.copy('data/lucid-sources.list', sources_list)
backup = os.path.join(tmpdir, "etc", "apt", "sources.list.apt-clone")
shutil.copy('data/natty-sources.list', backup)
clone._rewrite_sources_list(tmpdir, 'natty')
with open(sources_list) as fp:
# Tally the occurances of every source line.
from collections import defaultdict
tally = defaultdict(int)
for line in fp:
if line != '\n' and not line.startswith('#'):
tally[line] += 1
# There should not be any duplicate source lines.
for line, count in tally.items():
self.assertTrue(count == 1,
'"%s" occurred %d times.' % (line, count))
# Check for extras, others...
l = ((
'partner',
'deb http://archive.canonical.com/ubuntu natty partner\n'
), (
'extras', 'deb http://extras.ubuntu.com/ubuntu natty main\n'
), ('main',
'deb http://gb.archive.ubuntu.com/ubuntu/ natty main restricted\n'
))
for pocket, match in l:
fp.seek(0)
found = False
for line in fp:
if line == match:
found = True
self.assertTrue(
found, '%s repository not present or disabled.' % pocket)
def _save_state(self, with_dpkg_repack):
# setup mock
targetdir = self.tempdir
# test
clone = AptClone(cache_cls=MockAptCache)
sourcedir = "./data/mock-system"
clone.save_state(sourcedir,
targetdir,
with_dpkg_repack,
with_dpkg_status=True)
# verify that we got the tarfile
tarname = os.path.join(targetdir, clone.CLONE_FILENAME)
self.assertTrue(os.path.exists(tarname))
with tarfile.open(tarname) as tar:
#print(tar.getmembers())
# verify members in tar
members = [m.name for m in tar.getmembers()]
self.assertTrue("./etc/apt/sources.list" in members)
self.assertTrue("./var/lib/apt-clone/installed.pkgs" in members)
self.assertTrue("./var/lib/apt-clone/foreign.pkgs" in members)
self.assertTrue("./var/lib/apt-clone/extended_states" in members)
self.assertTrue("./var/lib/apt-clone/dpkg-status" in members)
self.assertTrue("./etc/apt/sources.list.d" in members)
self.assertTrue("./etc/apt/preferences.d" in members)
self.assertTrue("./etc/apt/preferences" in members)
if clone.not_downloadable:
self.assertEqual(clone.commands.repack_deb.called,
with_dpkg_repack)
# ensure we have no duplicates in the sources.list.d
sources_list_d = [
p for p in members if p.startswith("./etc/apt/sources.list.d")
]
self.assertEqual(
sorted(sources_list_d),
sorted([
'./etc/apt/sources.list.d',
'./etc/apt/sources.list.d/ubuntu-mozilla-daily-ppa-maverick.list'
]))
def test_clone_upgrade_synthetic(self):
""" test clone upgrade with on-the-fly generated chroots """
supported = distro_info.UbuntuDistroInfo().supported()
for meta in ["ubuntu-standard", "ubuntu-desktop", "kubuntu-desktop",
"xubuntu-desktop"]:
logging.info("testing %s" % meta)
old = self._create_fake_upgradable_root(supported[-2], meta=meta)
self.addCleanup(shutil.rmtree, old)
# create statefile based on the old data
with tarfile.open("lala.tar.gz", "w:gz") as state:
state.add(
os.path.join(old, "var", "lib", "apt-clone",
"installed.pkgs"),
arcname = "./var/lib/apt-clone/installed.pkgs")
# create new fake environment and try to upgrade
new = self._create_fake_upgradable_root(supported[-1], meta=meta)
self.addCleanup(shutil.rmtree, new)
cache = apt.Cache(rootdir=new)
clone = AptClone()
clone._restore_package_selection_in_cache("lala.tar.gz", cache, protect_installed=True)
self.assertFalse(cache[meta].marked_delete,
"package %s marked for removal" % meta)
self.assertTrue(len(cache.get_changes()) > 0)
def save_system_state(logdir):
# save package state to be able to re-create failures
try:
from apt_clone import AptClone
except ImportError:
logging.error("failed to import AptClone")
return
target = os.path.join(logdir, "apt-clone_system_state.tar.gz")
logging.debug("creating statefile: '%s'" % target)
# this file may contain sensitive data so ensure we create with the
# right umask
old_umask = os.umask(0066)
clone = AptClone()
clone.save_state(sourcedir="/", target=target, with_dpkg_status=True,
scrub_sources=True)
# reset umask
os.umask(old_umask)
# lspci output
try:
s=subprocess.Popen(["lspci","-nn"], stdout=subprocess.PIPE).communicate()[0]
open(os.path.join(logdir, "lspci.txt"), "w").write(s)
except OSError, e:
logging.debug("lspci failed: %s" % e)
def test_merge_sources(self):
clone = AptClone()
tmpdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmpdir)
sources_list = os.path.join(tmpdir, "etc", "apt", "sources.list")
os.makedirs(os.path.dirname(sources_list))
shutil.copy('data/lucid-sources.list', sources_list)
backup = os.path.join(tmpdir, "etc", "apt", "sources.list.apt-clone")
shutil.copy('data/natty-sources.list', backup)
clone._rewrite_sources_list(tmpdir, 'natty')
with open(sources_list) as fp:
# Tally the occurances of every source line.
from collections import defaultdict
tally = defaultdict(int)
for line in fp:
if line != '\n' and not line.startswith('#'):
tally[line] += 1
# There should not be any duplicate source lines.
for line, count in tally.items():
self.assertTrue(count == 1, '"%s" occurred %d times.'
% (line, count))
# Check for extras, others...
l = (('partner',
'deb http://archive.canonical.com/ubuntu natty partner\n'),
('extras',
'deb http://extras.ubuntu.com/ubuntu natty main\n'),
('main',
'deb http://gb.archive.ubuntu.com/ubuntu/ natty main restricted\n'))
for pocket, match in l:
fp.seek(0)
found = False
for line in fp:
if line == match:
found = True
self.assertTrue(found,
'%s repository not present or disabled.' % pocket)
class TestClone(unittest.TestCase):
def setUp(self):
self.apt_clone = AptClone()
self.test_sources_fname = "test-sources.list"
with open(self.test_sources_fname, "wb") as f:
f.write(u"""# äüö
deb http://mvo:[email protected]/ ubuntu main
""".encode("utf-8"))
def tearDown(self):
os.unlink(self.test_sources_fname)
def test_scrub_file_from_passwords(self):
"""Regression test for utf8 crash LP: #1309447"""
mock_tar = MockTar()
self.apt_clone._add_file_to_tar_with_password_check(
mock_tar, self.test_sources_fname, scrub=True,
arcname="some-archname")
# see if we got the expected data
self.assertNotIn("mvo:secret", mock_tar.data)
self.assertEqual(mock_tar.data, u"""# äüö
deb http://USERNAME:[email protected]/ ubuntu main
""")
def test_modified_conffiles(self):
clone = AptClone()
modified = clone._find_modified_conffiles("./data/mock-system")
self.assertEqual(modified,
set(["./data/mock-system/etc/conffile.modified"]))
def test_modified_conffiles(self):
clone = AptClone()
modified = clone._find_modified_conffiles("./data/mock-system")
self.assertEqual(
modified, set(["./data/mock-system/etc/conffile.modified"]))
