Exemplo n.º 1
0
    def librepo_target(self):
        pkg = self.pkg
        pkgdir = pkg.repo.pkgdir
        dnf.util.ensure_dir(pkgdir)

        target_dct = {
            'handle' : pkg.repo.get_handle(),
            'dest' : pkgdir,
            'resume' : True,
            'cbdata' : self,
            'progresscb' : self._progress_cb,
            'endcb' : self._end_cb,
            'mirrorfailurecb' : self._mirrorfail_cb,
        }
        target_dct.update(self._target_params())

        return librepo.PackageTarget(**target_dct)
    def test_download_packages_01(self):
        h = librepo.Handle()

        url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
        h.urls = [url]
        h.repotype = librepo.LR_YUMREPO

        pkgs = []
        pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
                                          handle=h,
                                          dest=self.tmpdir))

        librepo.download_packages(pkgs)

        for pkg in pkgs:
            self.assertTrue(pkg.err is None)
            self.assertTrue(os.path.isfile(pkg.local_path))
Exemplo n.º 3
0
    def test_download_packages_with_resume_02(self):
        # If download that should be resumed fails,
        # the original file should not be modified or deleted
        h = librepo.Handle()

        url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
        h.urls = [url]
        h.repotype = librepo.LR_YUMREPO
        h.allowedmirrorfailures = 1

        fn = os.path.join(self.tmpdir, "package.rpm")

        # Make fake unfinished download
        CONTENT = "0123456789"
        open(fn, "w").write(CONTENT)

        # Mark the file as it was downloaded by Librepo
        # Otherwise librepo refuse to resume
        try:
            xattr.setxattr(fn,
                           "user.Librepo.DownloadInProgress".encode("utf-8"),
                           "".encode("utf-8"))
        except IOError as err:
            if err.errno == 95:
                self.skipTest('extended attributes are not supported')
            raise

        # Now try to resume from bad URL
        pkgs = []
        pkgs.append(librepo.PackageTarget("bad_path.rpm",
                                          handle=h,
                                          dest=fn,
                                          resume=True,
                                          checksum_type=librepo.SHA256,
                                          checksum=config.PACKAGE_01_01_SHA256))

        # Download should fail (path is bad, the file doesn't exist)
        self.assertRaises(librepo.LibrepoException, librepo.download_packages,
                pkgs, failfast=True)

        # The package should exists (should not be removed or changed)
        pkg = pkgs[0]
        self.assertTrue(pkg.err)
        self.assertTrue(os.path.isfile(pkg.local_path))
        self.assertEqual(open(pkg.local_path, "rb").read(), CONTENT.encode('utf-8'))
Exemplo n.º 4
0
    def test_download_packages_with_baseurl(self):
        h = librepo.Handle()

        url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
        h.setopt(librepo.LRO_URLS, ["."])
        h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)

        pkgs = []
        pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
                                          handle=h,
                                          dest=self.tmpdir,
                                          base_url=url))

        librepo.download_packages(pkgs)

        pkg = pkgs[0]
        self.assertTrue(pkg.err is None)
        self.assertTrue(os.path.isfile(pkg.local_path))
Exemplo n.º 5
0
    def test_download_packages_with_bad_checksum(self):
        h = librepo.Handle()

        url = "%s%s" % (MOCKURL, config.REPO_YUM_01_PATH)
        h.setopt(librepo.LRO_URLS, [url])
        h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)

        pkgs = []
        pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
                                          handle=h,
                                          dest=self.tmpdir,
                                          checksum_type=librepo.SHA256,
                                          checksum="badchecksum"))

        librepo.download_packages(pkgs)

        pkg = pkgs[0]
        self.assertTrue(pkg.err)
    def test_download_packages_with_bad_checksum_with_failfast(self):
        h = librepo.Handle()

        url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
        h.urls = [url]
        h.repotype = librepo.LR_YUMREPO

        pkgs = []
        pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
                                          handle=h,
                                          dest=self.tmpdir,
                                          checksum_type=librepo.SHA256,
                                          checksum="badchecksum"))

        self.assertRaises(librepo.LibrepoException, librepo.download_packages,
                          pkgs, failfast=True)

        pkg = pkgs[0]
        self.assertTrue(pkg.err)
Exemplo n.º 7
0
 def download_package(self, location, checksum_type, checksum):
     if self.librepo_handle.local:
         local_path = os.path.join(self._root_path, location)
         logger.debug('Using package %s from local filesystem directly', local_path)
         return local_path
     logger.debug('Loading package %s from repo %s', location, self.name)
     target = librepo.PackageTarget(location,
             checksum_type=librepo.checksum_str_to_type(checksum_type),
             checksum=checksum,
             dest=self._root_path,
             handle=self.librepo_handle)
     librepo.download_packages([target])
     if target.err and target.err == 'Already downloaded':
         logger.debug('Already downloaded %s', target.local_path)
     elif target.err:
         raise PackageDownloadError('Failed to download %s from repo %s: %s'
                 % (location, self.name, target.err))
     else:
         logger.debug('Saved as %s', target.local_path)
     return target.local_path
Exemplo n.º 8
0
    def test_packagetarget_sanity_01(self):

        h = librepo.Handle()
        cbdata = {"a": "b"}

        def progresscb(a, b, c):
            return 0

        def endcb(a):
            return

        def mirrorfailurecb(a, b):
            return 0

        t = librepo.PackageTarget("foo",
                                  dest="bar",
                                  checksum_type=librepo.CHECKSUM_SHA512,
                                  checksum="xxx",
                                  expectedsize=123,
                                  base_url="basefoo",
                                  resume=True,
                                  progresscb=progresscb,
                                  cbdata=cbdata,
                                  handle=h,
                                  endcb=endcb,
                                  mirrorfailurecb=mirrorfailurecb)

        self.assertEqual(t.relative_url, "foo")
        self.assertEqual(t.dest, "bar")
        self.assertEqual(t.checksum_type, librepo.CHECKSUM_SHA512)
        self.assertEqual(t.checksum, "xxx")
        self.assertEqual(t.expectedsize, 123)
        self.assertEqual(t.base_url, "basefoo")
        self.assertEqual(t.resume, True)
        self.assertEqual(t.progresscb, progresscb)
        self.assertEqual(t.cbdata, cbdata)
        self.assertEqual(t.handle, h)
        self.assertEqual(t.endcb, endcb)
        self.assertEqual(t.mirrorfailurecb, mirrorfailurecb)
Exemplo n.º 9
0
    librepo.set_debug_log_handler(debug_function)

    # Prepare handle
    h = librepo.Handle()
    h.urls = [
        "http://mirror.karneval.cz/pub/linux/fedora/linux/releases/20/Fedora/x86_64/os/"
    ]
    h.repotype = librepo.YUMREPO

    # Prepare list of targets
    packages = []

    target = librepo.PackageTarget("Packages/a/abrt-2.1.9-1.fc20.x86_64.rpm",
                                   handle=h,
                                   dest="abrt.header",
                                   byterangestart=1384,
                                   byterangeend=100204)
    packages.append(target)

    librepo.download_packages(packages)

    for target in packages:
        print "### %s: %s" % (target.local_path, target.err or "OK")
        print "Relative URL:      ", target.relative_url
        print "Destination:       ", target.dest
        print "Base URL:          ", target.base_url
        print "Checksum type:     ", target.checksum_type
        print "Expected checksum: ", target.checksum
        print "Resume:            ", bool(target.resume)
        print "Local path:        ", target.local_path
Exemplo n.º 10
0
        print(msg)

    librepo.set_debug_log_handler(debug_function)

    # Prepare handle
    h = librepo.Handle()
    h.urls = ["http://beaker-project.org/yum/client-testing/Fedora21/"]
    h.repotype = librepo.YUMREPO

    # Prepare list of targets
    packages = []

    target = librepo.PackageTarget(
        "beaker-19.1-1.fc19.src.rpm",
        handle=h,
        checksum_type=librepo.SHA256,
        checksum=
        "3a07327702d15de2707518eb777c0837b2fc7c0cf4cb757331982f5ce6053867",
        resume=True)
    packages.append(target)

    target = librepo.PackageTarget("beaker-client-19.1-1.fc19.noarch.rpm",
                                   handle=h,
                                   expectedsize=381148)
    packages.append(target)

    target = librepo.PackageTarget("rhts-4.65-1.fc19.src.rpm", handle=h)
    packages.append(target)

    librepo.download_packages(packages)
Exemplo n.º 11
0
        if os.path.exists(filename):
            os.remove(filename)

    remove_pkg(LIBREPOPKG)
    remove_pkg(LAMEPKG)

    # Prepare list of targets
    packages = []

    # Prepare first target
    h1 = librepo.Handle()
    h1.metalinkurl = "https://mirrors.fedoraproject.org/metalink?repo=fedora-20&arch=x86_64"
    h1.repotype = librepo.YUMREPO
    h1.fastestmirror = True
    h1.fastestmirrorcache = CACHE
    target = librepo.PackageTarget("Packages/l/" + LIBREPOPKG, handle=h1)
    packages.append(target)

    # Prepare second target
    h2 = librepo.Handle()
    h2.mirrorlisturl = "http://mirrors.rpmfusion.org/mirrorlist?repo=free-fedora-19&arch=x86_64"
    h2.repotype = librepo.YUMREPO
    h2.fastestmirror = True
    h2.fastestmirrorcache = CACHE
    target = librepo.PackageTarget(LAMEPKG, handle=h2)
    packages.append(target)

    t = time.time()
    librepo.download_packages(packages)
    print "Download duration: {0}s\n".format((time.time() - t))
Exemplo n.º 12
0
            os.remove(CACHE)

    def remove_package():
        if os.path.exists(PACKAGE):
            print "Removing %s" % PACKAGE
            os.remove(PACKAGE)

    remove_cache()
    remove_package()

    print "\n1st run: Basic download"
    h = librepo.Handle()
    h.interruptible = True
    h.metalinkurl = METALINKURL
    h.repotype = librepo.YUMREPO
    packages = [librepo.PackageTarget(PACKAGEPATH+PACKAGE, handle=h)]
    t = time.time()
    librepo.download_packages(packages)
    t1 = time.time() - t
    print "### %s: %s (Time: %s)" % (
            packages[0].local_path, packages[0].err or "OK", t1)

    remove_package()

    print "\n2nd run: Download with fastestmirror - cache will be builded"
    h = librepo.Handle()
    h.interruptible = True
    h.metalinkurl = METALINKURL
    h.repotype = librepo.YUMREPO
    h.fastestmirror = True
    h.fastestmirrorcache = CACHE
    def test_download_packages_from_different_mirrors_1(self):
        cbdata = {
            "failed_urls": [],
        }

        def mirrorfailurecb(userdata, msg, url):
            cbdata["failed_urls"].append(url)
            return None

        h = librepo.Handle()

        url1 = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
        url2 = "%s%s" % (self.MOCKURL, config.REPO_YUM_03_PATH)
        url3 = "%s%s" % (self.MOCKURL, config.REPO_YUM_04_PATH)

        h.mirrorlist = "%s%s" % (self.MOCKURL, config.METALINK_MIRRORS_01)
        h.repotype = librepo.LR_YUMREPO
        h.maxparalleldownloads = 1
        h.fastestmirror = True

        pkgs = []
        for P in [
                config.PACKAGE_01_01, config.PACKAGE_03_01,
                config.PACKAGE_04_01
        ]:
            pkgs.append(
                librepo.PackageTarget(P,
                                      handle=h,
                                      dest=self.tmpdir,
                                      mirrorfailurecb=mirrorfailurecb))

        librepo.download_packages(pkgs, failfast=True)

        # the mirror list should be 1, 3, 4. The metalink file
        # defines preference order. This is the order in which
        # the URLs are listed in the file
        self.assertEqual(h.mirrors, [url1, url2, url3])

        # YUM 01 contains P_01
        # YUM 03 contains P_03
        # YUM_04 contains P_04
        # Mirror order of preference is 01 03, 04
        # When a package fails to download, librepo moves to the next mirror;
        # after a successfull download of the package librepo should continue trying
        # to download the remaining packages from the first mirror. e.g.
        # always try to download from the fastest mirror containing the package

        # Expected download sequence (restart from fastest mirror):
        # - P_01 from YUM_01 - PASS
        # - P_03 from YUM_01 - FAIL
        # - P_03 from YUM_03 - PASS
        # - P_04 from YUM_04 - FAIL
        # - P_04 from YUM_03 - FAIL
        # - P_04 from YUM_04 - PASS
        self.assertEqual(len(cbdata["failed_urls"]), 3)
        self.assertEqual(cbdata["failed_urls"][0], url1 + config.PACKAGE_03_01)
        self.assertEqual(cbdata["failed_urls"][1], url1 + config.PACKAGE_04_01)
        self.assertEqual(cbdata["failed_urls"][2], url2 + config.PACKAGE_04_01)

        # assert that all packages have been downloaded
        for pkg in pkgs:
            self.assertTrue(pkg.err is None)
            self.assertTrue(os.path.isfile(pkg.local_path))
    def test_download_packages_with_resume_02(self):
        return # This test causes issues on Fedora rawhide (F26)
               # Flask/werkzeug/SocketServer/socket fails on Broken pipe
               # and causes next test (test_download_packages_without_handle)
               # to always fail

        # If download that should be resumed fails,
        # the original file should not be modified or deleted
        h = librepo.Handle()

        url = "%s%s" % (self.MOCKURL, config.REPO_YUM_01_PATH)
        h.urls = [url]
        h.repotype = librepo.LR_YUMREPO

        fn = os.path.join(self.tmpdir, "package.rpm")

        # Download first 10 bytes of the package
        pkgs = []
        pkgs.append(librepo.PackageTarget(config.PACKAGE_01_01,
                                          handle=h,
                                          dest=fn,
                                          resume=False,
                                          byterangeend=9))

        librepo.download_packages(pkgs)
        pkg = pkgs[0]
        self.assertTrue(pkg.err is None)
        self.assertTrue(os.path.isfile(pkg.local_path))
        self.assertEqual(os.path.getsize(pkg.local_path), 10)
        fchksum = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()

        # Mark the file as it was downloaded by Librepo
        # Otherwise librepo refuse to resume
        try:
            xattr.setxattr(pkg.local_path,
                           "user.Librepo.DownloadInProgress".encode("utf-8"),
                           "".encode("utf-8"))
        except IOError as err:
            if err.errno == 95:
                self.skipTest('extended attributes are not supported')
            raise

        # Now try to resume from bad URL
        pkgs = []
        pkgs.append(librepo.PackageTarget("bad_path.rpm",
                                          handle=h,
                                          dest=fn,
                                          resume=True,
                                          checksum_type=librepo.SHA256,
                                          checksum=config.PACKAGE_01_01_SHA256))

        # Download should fail (path is bad, the file doesn't exist)
        self.assertRaises(librepo.LibrepoException, librepo.download_packages,
                pkgs, failfast=True)

        # The package should exists (should not be removed or changed)
        pkg = pkgs[0]
        self.assertTrue(pkg.err)
        self.assertTrue(os.path.isfile(pkg.local_path))
        self.assertEqual(os.path.getsize(pkg.local_path), 10)
        fchksum_new = hashlib.md5(open(pkg.local_path, "rb").read()).hexdigest()
        self.assertEqual(fchksum, fchksum_new)
Exemplo n.º 15
0
        print msg

    librepo.set_debug_log_handler(debug_function)

    # Prepare handle
    h = librepo.Handle()
    h.urls = ["http://beaker-project.org/yum/client-testing/Fedora19/"]
    h.repotype = librepo.YUMREPO

    # Prepare list of targets
    packages = []

    target = librepo.PackageTarget(
        "beaker-0.14.0-1.fc18.src.rpm",
        handle=h,
        checksum_type=librepo.SHA256,
        checksum=
        "737c974110914a073fb6c736cd7021b0d844c9e47e7d21e37d687dbc86d36538",
        resume=True)
    packages.append(target)

    target = librepo.PackageTarget("beaker-client-0.14.1-1.fc18.noarch.rpm",
                                   handle=h,
                                   expectedsize=325144)
    packages.append(target)

    target = librepo.PackageTarget("rhts-4.56-1.fc17.src.rpm", handle=h)
    packages.append(target)

    librepo.download_packages(packages)
Exemplo n.º 16
0
            print "Already exists"
        else:
            print "Error: %s" % msg

    def mirrorfailurecb(data, msg, url):
        print "MirrorFailureCb: Download of %s from %s failed with: %s" % \
              (data, url, msg)

    # Prepare list of targets
    packages = []

    target = librepo.PackageTarget(
        "beaker-0.14.0-1.fc18.src.rpm",
        handle=h,
        checksum_type=librepo.SHA256,
        checksum=
        "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
        resume=True,
        cbdata="beaker-0.14.0-1.fc18.src.rpm",
        endcb=endcb,
        mirrorfailurecb=mirrorfailurecb)
    packages.append(target)

    target = librepo.PackageTarget(
        "beaker-0.13.2-1.fc17.noarch.rpm",
        handle=h,
        checksum_type=librepo.SHA256,
        checksum="foobar",
        cbdata="beaker-0.13.2-1.fc17.noarch.rpm (bad checksum)",
        endcb=endcb,
        mirrorfailurecb=mirrorfailurecb)
    packages.append(target)