def save_system_state(logdir):
    # save package state to be able to re-create failures
    try:
        from apt_clone import AptClone
    except ImportError:
        logging.error("failed to import AptClone")
        return
    target = os.path.join(logdir, "apt-clone_system_state.tar.gz")
    logging.debug("creating statefile: '%s'" % target)
    # this file may contain sensitive data so ensure we create with the
    # right umask
    old_umask = os.umask(0066)
    clone = AptClone()
    clone.save_state(sourcedir="/",
                     target=target,
                     with_dpkg_status=True,
                     scrub_sources=True)
    # reset umask
    os.umask(old_umask)
    # lspci output
    try:
        s = subprocess.Popen(["lspci", "-nn"],
                             stdout=subprocess.PIPE).communicate()[0]
        open(os.path.join(logdir, "lspci.txt"), "w").write(s)
    except OSError, e:
        logging.debug("lspci failed: %s" % e)
Example #2
0
 def test_restore_state_on_new_distro_release_livecd(self, mock_lowlevel):
     """
     test lucid -> maverick apt-clone-ugprade as if it will be used
     from a live cd based upgrader
     """
     # setup mock for dpkg -i
     mock_lowlevel.install_debs.return_value = True
     # create target dir
     targetdir = self.tempdir
     # status file from maverick (to simulate running on a maverick live-cd)
     with open("./data/dpkg-status/dpkg-status-ubuntu-maverick",
               "rb") as fp:
         s = fp.read().decode("utf8")
     s = s.replace(
         "Architecture: i386",
         "Architecture: %s" % apt_pkg.config.find("Apt::Architecture"))
     path = os.path.join(targetdir, "var/lib/dpkg", "status")
     with open(path, "wb",) as fp:
         fp.write(s.encode("utf-8"))
     # test upgrade clone from lucid system to maverick
     clone = AptClone(cache_cls=MockAptCache)
     clone.restore_state(
         "./data/apt-state-ubuntu-lucid.tar.gz",
         targetdir,
         new_distro="maverick")
     sources_list = os.path.join(targetdir, "etc","apt","sources.list")
     self.assertTrue(os.path.exists(sources_list))
     with open(sources_list) as fp:
         self.assertTrue("maverick" in fp.read())
     with open(sources_list) as fp:
         self.assertFalse("lucid" in fp.read())
Example #3
0
 def test_clone_upgrade_synthetic(self):
     """ test clone upgrade with on-the-fly generated chroots """
     supported = distro_info.UbuntuDistroInfo().supported()
     for meta in [
             "ubuntu-standard", "ubuntu-desktop", "kubuntu-desktop",
             "xubuntu-desktop"
     ]:
         logging.info("testing %s" % meta)
         old = self._create_fake_upgradable_root(supported[-2], meta=meta)
         self.addCleanup(shutil.rmtree, old)
         # create statefile based on the old data
         with tarfile.open("lala.tar.gz", "w:gz") as state:
             state.add(os.path.join(old, "var", "lib", "apt-clone",
                                    "installed.pkgs"),
                       arcname="./var/lib/apt-clone/installed.pkgs")
         # create new fake environment and try to upgrade
         new = self._create_fake_upgradable_root(supported[-1], meta=meta)
         self.addCleanup(shutil.rmtree, new)
         cache = apt.Cache(rootdir=new)
         clone = AptClone()
         clone._restore_package_selection_in_cache("lala.tar.gz",
                                                   cache,
                                                   protect_installed=True)
         self.assertFalse(cache[meta].marked_delete,
                          "package %s marked for removal" % meta)
         self.assertTrue(len(cache.get_changes()) > 0)
Example #4
0
 def test_restore_state_simulate_with_new_release(self):
     #apt_pkg.config.set("Debug::PkgProblemResolver", "1")
     apt_pkg.config.set("Dir::state::status",
                        "./data/dpkg-status/dpkg-status-ubuntu-maverick")
     clone = AptClone()
     missing = clone.simulate_restore_state(
         "./data/apt-state-ubuntu-lucid.tar.gz", "maverick")
Example #5
0
    def test_restore_state_simulate(self):
        clone = AptClone()
        supported = distro_info.UbuntuDistroInfo().supported()

        missing = clone.simulate_restore_state("./data/apt-state.tar.gz", new_distro=supported[-1], exclude_pkgs=[])
        # missing, because clone does not have universe enabled
        self.assertEqual(list(missing), ['accerciser'])
Example #6
0
 def test_restore_state_on_new_distro_release_livecd(self, mock_lowlevel):
     """
     test lucid -> maverick apt-clone-ugprade as if it will be used
     from a live cd based upgrader
     """
     # setup mock for dpkg -i
     mock_lowlevel.install_debs.return_value = True
     # create target dir
     targetdir = self.tempdir
     # status file from maverick (to simulate running on a maverick live-cd)
     with open("./data/dpkg-status/dpkg-status-ubuntu-maverick",
               "rb") as fp:
         s = fp.read().decode("utf8")
     s = s.replace(
         "Architecture: i386",
         "Architecture: %s" % apt_pkg.config.find("Apt::Architecture"))
     path = os.path.join(targetdir, "var/lib/dpkg", "status")
     with open(
             path,
             "wb",
     ) as fp:
         fp.write(s.encode("utf-8"))
     # test upgrade clone from lucid system to maverick
     clone = AptClone(cache_cls=MockAptCache)
     clone.restore_state("./data/apt-state-ubuntu-lucid.tar.gz",
                         targetdir,
                         new_distro="maverick")
     sources_list = os.path.join(targetdir, "etc", "apt", "sources.list")
     self.assertTrue(os.path.exists(sources_list))
     with open(sources_list) as fp:
         self.assertTrue("maverick" in fp.read())
     with open(sources_list) as fp:
         self.assertFalse("lucid" in fp.read())
Example #7
0
 def _save_state(self, with_dpkg_repack):
     # setup mock
     targetdir = self.tempdir
     # test
     clone = AptClone(cache_cls=MockAptCache)
     sourcedir = "./data/mock-system"
     clone.save_state(sourcedir, targetdir, with_dpkg_repack, with_dpkg_status=True)
     # verify that we got the tarfile
     tarname = os.path.join(targetdir, clone.CLONE_FILENAME)
     self.assertTrue(os.path.exists(tarname))
     with tarfile.open(tarname) as tar:
         #print(tar.getmembers())
         # verify members in tar
         members = [m.name for m in tar.getmembers()]
     self.assertTrue("./etc/apt/sources.list" in members)
     self.assertTrue("./var/lib/apt-clone/installed.pkgs" in members)
     self.assertTrue("./var/lib/apt-clone/foreign.pkgs" in members)
     self.assertTrue("./var/lib/apt-clone/extended_states" in members)
     self.assertTrue("./var/lib/apt-clone/dpkg-status" in members)
     self.assertTrue("./etc/apt/sources.list.d" in members)
     self.assertTrue("./etc/apt/preferences.d" in members)
     self.assertTrue("./etc/apt/preferences" in members)
     if clone.not_downloadable:
         self.assertEqual(clone.commands.repack_deb.called, with_dpkg_repack)
     # ensure we have no duplicates in the sources.list.d
     sources_list_d = [p for p in members
                       if p.startswith("./etc/apt/sources.list.d")]
     self.assertEqual(
         sorted(sources_list_d),
         sorted(
             ['./etc/apt/sources.list.d',
              './etc/apt/sources.list.d/ubuntu-mozilla-daily-ppa-maverick.list']))
Example #8
0
 def test_restore_state_simulate_with_new_release(self):
     #apt_pkg.config.set("Debug::PkgProblemResolver", "1")
     apt_pkg.config.set(
         "Dir::state::status",
         "./data/dpkg-status/dpkg-status-ubuntu-maverick")
     clone = AptClone()
     missing = clone.simulate_restore_state(
         "./data/apt-state-ubuntu-lucid.tar.gz", "maverick")
Example #9
0
 def test_clone_upgrade_regression(self):
     """ regression test against known installs """
     new = self._create_fake_upgradable_root("natty", meta="ubuntu-desktop")
     self.addCleanup(shutil.rmtree, new)
     cache = apt.Cache(rootdir=new)
     clone = AptClone()
     clone._restore_package_selection_in_cache(
         "./data/regression/apt-clone-state-ubuntu.tar.gz", cache)
     self.assertTrue(len(cache.get_changes()) > 0)
Example #10
0
 def test_clone_upgrade_regression(self):
     """ regression test against known installs """
     new = self._create_fake_upgradable_root("natty", meta="ubuntu-desktop")
     self.addCleanup(shutil.rmtree, new)
     cache = apt.Cache(rootdir=new)
     clone = AptClone()
     clone._restore_package_selection_in_cache(
         "./data/regression/apt-clone-state-ubuntu.tar.gz", cache)
     self.assertTrue(len(cache.get_changes()) > 0)
Example #11
0
    def test_restore_state_simulate(self):
        clone = AptClone()
        supported = distro_info.UbuntuDistroInfo().supported()

        missing = clone.simulate_restore_state("./data/apt-state.tar.gz",
                                               new_distro=supported[-1],
                                               exclude_pkgs=[])
        # missing, because clone does not have universe enabled
        self.assertEqual(list(missing), ['accerciser'])
Example #12
0
 def test_restore_state(self, mock_lowlevel):
     # setup mock
     mock_lowlevel.install_debs.return_value = True
     targetdir = self.tempdir
     # test
     clone = AptClone(cache_cls=MockAptCache)
     clone.restore_state(
         "./data/apt-state_chroot_with_vim.tar.gz", targetdir)
     self.assertTrue(
         os.path.exists(os.path.join(targetdir, "etc","apt","sources.list")))
Example #13
0
 def test_restore_state_with_not_downloadable_debs(self, mock_lowlevel):
     # setup mock
     mock_lowlevel.install_debs.return_value = True
     targetdir = self.tempdir
     # test
     clone = AptClone(cache_cls=MockAptCache)
     clone.restore_state(
         "./data/apt-state_with_not_downloadable_debs.tar.gz", targetdir)
     self.assertTrue(
         os.path.exists(
             os.path.join(targetdir, "var", "lib", "apt-clone", "debs", "foo.deb")))
Example #14
0
 def test_restore_state(self, mock_lowlevel):
     # setup mock
     mock_lowlevel.install_debs.return_value = True
     targetdir = self.tempdir
     # test
     clone = AptClone(cache_cls=MockAptCache)
     clone.restore_state("./data/apt-state_chroot_with_vim.tar.gz",
                         targetdir)
     self.assertTrue(
         os.path.exists(
             os.path.join(targetdir, "etc", "apt", "sources.list")))
Example #15
0
 def test_restore_state_with_not_downloadable_debs(self, mock_lowlevel):
     # setup mock
     mock_lowlevel.install_debs.return_value = True
     targetdir = self.tempdir
     # test
     clone = AptClone(cache_cls=MockAptCache)
     clone.restore_state(
         "./data/apt-state_with_not_downloadable_debs.tar.gz", targetdir)
     self.assertTrue(
         os.path.exists(
             os.path.join(targetdir, "var", "lib", "apt-clone", "debs",
                          "foo.deb")))
Example #16
0
    def setUp(self):
        self.apt_clone = AptClone()
        self.test_sources_fname = "test-sources.list"
        with open(self.test_sources_fname, "wb") as f:
            f.write(u"""# äüö
deb http://mvo:[email protected]/ ubuntu main
""".encode("utf-8"))
def save_system_state(logdir):
    # save package state to be able to re-create failures
    try:
        from apt_clone import AptClone
    except ImportError:
        logging.error("failed to import AptClone")
        return
    target = os.path.join(logdir, "apt-clone_system_state.tar.gz")
    logging.debug("creating statefile: '%s'" % target)
    clone = AptClone()
    clone.save_state(sourcedir="/", target=target, with_dpkg_status=True)
    # lspci output
    try:
        s=subprocess.Popen(["lspci","-nn"], stdout=subprocess.PIPE).communicate()[0]
        open(os.path.join(logdir, "lspci.txt"), "w").write(s)
    except OSError, e:
        logging.debug("lspci failed: %s" % e)
Example #18
0
 def test_unowned_in_etc(self):
     # test in mock environement
     apt_pkg.config.set("Dir::state::status",
                        "./data/mock-system/var/lib/dpkg/status")
     clone = AptClone()
     unowned = clone._find_unowned_in_etc("./data/mock-system")
     self.assertFalse("/etc/conffile.modified" in unowned)
     self.assertFalse("/etc/conffile.not-modified" in unowned)
     self.assertTrue("/etc/unowned-file" in unowned)
     # test on the real system and do very light checks
     apt_pkg.config.set("Dir::state::status", "/var/lib/dpkg/status")
     unowned = clone._find_unowned_in_etc()
     #print(unowned)
     self.assertNotEqual(unowned, set())
     # negative test, is created by the installer
     self.assertTrue("/etc/apt/sources.list" in unowned)
     # postivie test, belongs to base-files
     self.assertFalse("/etc/issue" in unowned)
Example #19
0
    def test_real(self):
        if os.getuid() != 0:
            print("Skipping because uid != 0")
            return
        # do it
        target = "./test-chroot"
        if not os.path.exists(target):
            os.mkdir(target)
            subprocess.call(["debootstrap", "--arch=i386",
                             "maverick", target])
        # force i386
        with open(os.path.join(target, "etc/apt/apt.conf"), "w") as fp:
            fp.write('APT::Architecture "i386";')

        # restore
        clone = AptClone()
        clone.restore_state(
            "./data/apt-state_chroot_with_vim.tar.gz", target, "maverick")
Example #20
0
 def test_unowned_in_etc(self):
     # test in mock environement
     apt_pkg.config.set(
         "Dir::state::status",
         "./data/mock-system/var/lib/dpkg/status")
     clone = AptClone()
     unowned = clone._find_unowned_in_etc("./data/mock-system")
     self.assertFalse("/etc/conffile.modified" in unowned)
     self.assertFalse("/etc/conffile.not-modified" in unowned)
     self.assertTrue("/etc/unowned-file" in unowned)
     # test on the real system and do very light checks
     apt_pkg.config.set(
         "Dir::state::status",
         "/var/lib/dpkg/status")
     unowned = clone._find_unowned_in_etc()
     #print(unowned)
     self.assertNotEqual(unowned, set())
     # negative test, is created by the installer
     self.assertTrue("/etc/apt/sources.list" in unowned)
     # postivie test, belongs to base-files
     self.assertFalse("/etc/issue" in unowned)
Example #21
0
    def test_merge_sources(self):
        clone = AptClone()
        tmpdir = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, tmpdir)
        sources_list = os.path.join(tmpdir, "etc", "apt", "sources.list")
        os.makedirs(os.path.dirname(sources_list))
        shutil.copy('data/lucid-sources.list', sources_list)
        backup = os.path.join(tmpdir, "etc", "apt", "sources.list.apt-clone")
        shutil.copy('data/natty-sources.list', backup)
        clone._rewrite_sources_list(tmpdir, 'natty')
        with open(sources_list) as fp:
            # Tally the occurances of every source line.
            from collections import defaultdict
            tally = defaultdict(int)
            for line in fp:
                if line != '\n' and not line.startswith('#'):
                    tally[line] += 1
            # There should not be any duplicate source lines.
            for line, count in tally.items():
                self.assertTrue(count == 1,
                                '"%s" occurred %d times.' % (line, count))

            # Check for extras, others...
            l = ((
                'partner',
                'deb http://archive.canonical.com/ubuntu natty partner\n'
            ), (
                'extras', 'deb http://extras.ubuntu.com/ubuntu natty main\n'
            ), ('main',
                'deb http://gb.archive.ubuntu.com/ubuntu/ natty main restricted\n'
                ))
            for pocket, match in l:
                fp.seek(0)
                found = False
                for line in fp:
                    if line == match:
                        found = True
                self.assertTrue(
                    found, '%s repository not present or disabled.' % pocket)
Example #22
0
 def _save_state(self, with_dpkg_repack):
     # setup mock
     targetdir = self.tempdir
     # test
     clone = AptClone(cache_cls=MockAptCache)
     sourcedir = "./data/mock-system"
     clone.save_state(sourcedir,
                      targetdir,
                      with_dpkg_repack,
                      with_dpkg_status=True)
     # verify that we got the tarfile
     tarname = os.path.join(targetdir, clone.CLONE_FILENAME)
     self.assertTrue(os.path.exists(tarname))
     with tarfile.open(tarname) as tar:
         #print(tar.getmembers())
         # verify members in tar
         members = [m.name for m in tar.getmembers()]
     self.assertTrue("./etc/apt/sources.list" in members)
     self.assertTrue("./var/lib/apt-clone/installed.pkgs" in members)
     self.assertTrue("./var/lib/apt-clone/foreign.pkgs" in members)
     self.assertTrue("./var/lib/apt-clone/extended_states" in members)
     self.assertTrue("./var/lib/apt-clone/dpkg-status" in members)
     self.assertTrue("./etc/apt/sources.list.d" in members)
     self.assertTrue("./etc/apt/preferences.d" in members)
     self.assertTrue("./etc/apt/preferences" in members)
     if clone.not_downloadable:
         self.assertEqual(clone.commands.repack_deb.called,
                          with_dpkg_repack)
     # ensure we have no duplicates in the sources.list.d
     sources_list_d = [
         p for p in members if p.startswith("./etc/apt/sources.list.d")
     ]
     self.assertEqual(
         sorted(sources_list_d),
         sorted([
             './etc/apt/sources.list.d',
             './etc/apt/sources.list.d/ubuntu-mozilla-daily-ppa-maverick.list'
         ]))
Example #23
0
 def test_clone_upgrade_synthetic(self):
     """ test clone upgrade with on-the-fly generated chroots """
     supported = distro_info.UbuntuDistroInfo().supported()
     for meta in ["ubuntu-standard", "ubuntu-desktop", "kubuntu-desktop",
                  "xubuntu-desktop"]:
         logging.info("testing %s" % meta)
         old = self._create_fake_upgradable_root(supported[-2], meta=meta)
         self.addCleanup(shutil.rmtree, old)
         # create statefile based on the old data
         with tarfile.open("lala.tar.gz", "w:gz") as state:
             state.add(
                 os.path.join(old, "var", "lib", "apt-clone",
                              "installed.pkgs"),
                 arcname = "./var/lib/apt-clone/installed.pkgs")
         # create new fake environment and try to upgrade
         new = self._create_fake_upgradable_root(supported[-1], meta=meta)
         self.addCleanup(shutil.rmtree, new)
         cache = apt.Cache(rootdir=new)
         clone = AptClone()
         clone._restore_package_selection_in_cache("lala.tar.gz", cache, protect_installed=True)
         self.assertFalse(cache[meta].marked_delete,
                          "package %s marked for removal" % meta)
         self.assertTrue(len(cache.get_changes()) > 0)
def save_system_state(logdir):
    # save package state to be able to re-create failures
    try:
        from apt_clone import AptClone
    except ImportError:
        logging.error("failed to import AptClone")
        return
    target = os.path.join(logdir, "apt-clone_system_state.tar.gz")
    logging.debug("creating statefile: '%s'" % target)
    # this file may contain sensitive data so ensure we create with the
    # right umask
    old_umask = os.umask(0066)
    clone = AptClone()
    clone.save_state(sourcedir="/", target=target, with_dpkg_status=True,
        scrub_sources=True)
    # reset umask
    os.umask(old_umask)
    # lspci output
    try:
        s=subprocess.Popen(["lspci","-nn"], stdout=subprocess.PIPE).communicate()[0]
        open(os.path.join(logdir, "lspci.txt"), "w").write(s)
    except OSError, e:
        logging.debug("lspci failed: %s" % e)
Example #25
0
    def test_merge_sources(self):
        clone = AptClone()
        tmpdir = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, tmpdir)
        sources_list = os.path.join(tmpdir, "etc", "apt", "sources.list")
        os.makedirs(os.path.dirname(sources_list))
        shutil.copy('data/lucid-sources.list', sources_list)
        backup = os.path.join(tmpdir, "etc", "apt", "sources.list.apt-clone")
        shutil.copy('data/natty-sources.list', backup)
        clone._rewrite_sources_list(tmpdir, 'natty')
        with open(sources_list) as fp:
            # Tally the occurances of every source line.
            from collections import defaultdict
            tally = defaultdict(int)
            for line in fp:
                if line != '\n' and not line.startswith('#'):
                    tally[line] += 1
            # There should not be any duplicate source lines.
            for line, count in tally.items():
                self.assertTrue(count == 1, '"%s" occurred %d times.'
                                % (line, count))

            # Check for extras, others...
            l = (('partner',
                  'deb http://archive.canonical.com/ubuntu natty partner\n'),
                 ('extras',
                  'deb http://extras.ubuntu.com/ubuntu natty main\n'),
                 ('main',
                  'deb http://gb.archive.ubuntu.com/ubuntu/ natty main restricted\n'))
            for pocket, match in l:
                fp.seek(0)
                found = False
                for line in fp:
                    if line == match:
                        found = True
                self.assertTrue(found,
                        '%s repository not present or disabled.' % pocket)
Example #26
0
class TestClone(unittest.TestCase):

    def setUp(self):
        self.apt_clone = AptClone()
        self.test_sources_fname = "test-sources.list"
        with open(self.test_sources_fname, "wb") as f:
            f.write(u"""# äüö
deb http://mvo:[email protected]/ ubuntu main
""".encode("utf-8"))

    def tearDown(self):
        os.unlink(self.test_sources_fname)

    def test_scrub_file_from_passwords(self):
        """Regression test for utf8 crash LP: #1309447"""
        mock_tar = MockTar()
        self.apt_clone._add_file_to_tar_with_password_check(
            mock_tar, self.test_sources_fname, scrub=True, 
            arcname="some-archname")
        # see if we got the expected data
        self.assertNotIn("mvo:secret", mock_tar.data)
        self.assertEqual(mock_tar.data, u"""# äüö
deb http://USERNAME:[email protected]/ ubuntu main
""")
Example #27
0
 def test_modified_conffiles(self):
     clone = AptClone()
     modified = clone._find_modified_conffiles("./data/mock-system")
     self.assertEqual(modified,
                      set(["./data/mock-system/etc/conffile.modified"]))
Example #28
0
 def test_modified_conffiles(self):
     clone = AptClone()
     modified = clone._find_modified_conffiles("./data/mock-system")
     self.assertEqual(
         modified, set(["./data/mock-system/etc/conffile.modified"]))