Beispiel #1
0
    def test_store_rpm_deps(self):
        """
        """

        # add required stuff to db
        arch = Arch()
        arch.name = "noarch"
        self.db.session.add(arch)

        build = Build()
        build.base_package_name = "sample"
        build.version = "1"
        build.release = "1"
        build.epoch = "0"

        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build

        self.db.session.add(pkg)
        self.db.session.flush()

        # save sample rpm
        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg.save_lob("package", sample, truncate=True)

        # get dependencies
        res = store_rpm_deps(self.db, pkg, nogpgcheck=True)
        self.assertIs(res, True)

        expected_deps = [
            ("PROVIDES", "sample", 8),
            ("PROVIDES", "/sample", 0),
            ("PROVIDES", "nothing-new", 0),
            (
                "REQUIRES",
                "nothing",
                0,
            ),
            (
                "CONFLICTS",
                "surprisingly-nothing",
                0,
            ),
        ]

        found_deps = []

        for dep in self.db.session.query(PackageDependency).all():
            found_deps.append((dep.type, dep.name, dep.flags))

        for dep in expected_deps:
            self.assertIn(dep, found_deps)
Beispiel #2
0
    def restore_package_deps(self):
        print('Restoring package dependencies from rpms')
        for package in self.ses.query(Package):
            store_rpm_deps(self.db, package)

        self.ses.commit()
Beispiel #3
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if cmdline.match_repos not in repo.name:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if repo.opsys_list:  # parametrized
                self.log_info("Processing parametrized repo '{0}'"
                              .format(repo.name))

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture, skipping")
                    continue
                try:
                    repo_instances += list(self._get_parametrized_variants(repo))
                except: # pylint: disable=bare-except
                    self.log_error("No valid mirror for repository '{0}', skipping"
                                   .format(repo.name))
                    continue


            elif repo.opsysrelease_list:
                self.log_info("Processing repo '{0}' assigned with OpSysRelease"
                              .format(repo.name))

                if not repo.arch_list:
                    self.log_error("OpSysRelease repository is not assigned"
                                   " with an architecture, skipping")
                    continue
                try:
                    repo_instances += list(self._get_opsysrelease_variants(repo))
                except: # pylint: disable=bare-except
                    self.log_error("No valid mirror for repository '{0}', skipping"
                                   .format(repo.name))
                    continue

            else:
                if any('$' in url.url for url in repo.url_list):
                    self.log_error("No operating system assigned to"
                                   "parametrized repo '{0}', skipping".format(repo.name))
                    continue
                for arch in repo.arch_list:
                    try:
                        repo_instance = {
                            'instance' : repo_types[repo.type](
                                repo.name,
                                [url.url for url in repo.url_list]),
                            'opsys' : None,
                            'release' : None,
                            'arch' : arch.name,
                            'nogpgcheck': repo.nogpgcheck}
                        repo_instances.append(repo_instance)
                    except: # pylint: disable=bare-except
                        self.log_error("No valid mirror for repository '{0}', skipping"
                                       .format(repo.name))
                        continue

        cmdline.name_prefix = cmdline.name_prefix.lower()
        architectures = dict((x.name, x) for x in get_archs(db))
        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'"
                          .format(repo_instance['instance'].name,
                                  repo_instance['instance'].urls))

            pkglist = \
                repo_instance['instance'].list_packages(list(architectures.keys()))
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}"
                               .format(num + 1, total, pkg["name"]))

                if not pkg["name"].lower().startswith(cmdline.name_prefix):
                    self.log_debug("Skipped package {0}"
                                   .format(pkg["name"]))
                    continue
                arch = architectures.get(pkg["arch"], None)
                if not arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(pkg["arch"]))

                    continue

                repo_arch = architectures.get(repo_instance["arch"], None)
                if not repo_arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(repo_instance["arch"]))

                    continue

                build = (db.session.query(Build)
                         .filter(Build.base_package_name ==
                                 pkg["base_package_name"])
                         .filter(Build.version == pkg["version"])
                         .filter(Build.release == pkg["release"])
                         .filter(Build.epoch == pkg["epoch"])
                         .first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)
                    db.session.flush()

                build_arch = (db.session.query(BuildArch)
                              .filter(BuildArch.build_id == build.id)
                              .filter(BuildArch.arch_id == arch.id)
                              .first())

                if not build_arch:
                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                build_opsysrelease_arch = (
                    db.session.query(BuildOpSysReleaseArch)
                    .join(Build)
                    .join(OpSysRelease)
                    .join(Arch)
                    .filter(Build.id == build.id)
                    .filter(OpSys.name == repo_instance['opsys'])
                    .filter(OpSysRelease.version == repo_instance['release'])
                    .filter(Arch.name == repo_instance['arch'])
                    .first())

                if not build_opsysrelease_arch and repo_instance['release'] and repo_instance['opsys']:
                    self.log_info("Adding link between build {0}-{1} "
                                  "operating system '{2}', release '{3} and "
                                  "architecture {4}".format(pkg["base_package_name"],
                                                            pkg["version"], repo_instance['opsys'],
                                                            repo_instance['release'], repo_instance['arch']))

                    opsysrelease = (
                        db.session.query(OpSysRelease)
                        .filter(OpSys.name == repo_instance['opsys'])
                        .filter(OpSysRelease.version == repo_instance['release'])
                        .first())

                    bosra = BuildOpSysReleaseArch()
                    bosra.build = build
                    bosra.opsysrelease = opsysrelease
                    bosra.arch = repo_arch

                    db.session.add(bosra)
                    db.session.flush()

                package = (db.session.query(Package)
                           .filter(Package.name == pkg["name"])
                           .filter(Package.pkgtype == pkg["type"])
                           .filter(Package.build == build)
                           .filter(Package.arch == arch)
                           .first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attempts"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo_instance['nogpgcheck'])

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc: # pylint: disable=broad-except
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))
Beispiel #4
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if repo.opsys_list:  # parametrized
                self.log_info("Processing parametrized repo '{0}'"
                              .format(repo.name))

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture")
                    return 1
                try:
                    repo_instances += list(self._get_parametrized_variants(repo))
                except:
                    self.log_error("No valid mirror for repository {0}"
                                                    .format(repo.name))
                    return 1


            elif repo.opsysrelease_list:
                self.log_info("Processing repo '{0}' assigned with OpSysRelease"
                          .format(repo.name))

                if not repo.arch_list:
                    self.log_error("OpSysRelease repository is not assigned"
                                   " with an architecture")
                    return 1
                try:
                    repo_instances += list(self._get_opsysrelease_variants(repo))
                except:
                    self.log_error("No valid mirror for repository {0}"
                                                    .format(repo.name))
                    return 1

            else:
                if any('$' in url.url for url in repo.url_list):
                    self.log_error("No operating system assigned to"
                            "parametrized repo '{0}".format(repo.name))
                    return 1
                for arch in repo.arch_list:
                    try:
                        repo_instance = {
                                'instance' : repo_types[repo.type](repo.name,
                                            [url.url for url in repo.url_list]),
                                'opsys' : None,
                                'release' : None,
                                'arch' : arch.name}
                        repo_instances.append(repo_instance)
                    except:
                        self.log_error("No valid mirror for repository {0}"
                                                        .format(repo.name))
                        return 1

        cmdline.name_prefix = cmdline.name_prefix.lower()
        architectures = dict((x.name, x) for x in get_archs(db))
        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'"
                          .format(repo_instance['instance'].name,
                              repo_instance['instance'].urls))

            pkglist = \
                repo_instance['instance'].list_packages(architectures.keys())
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}"
                               .format(num + 1, total, pkg["name"]))

                if not pkg["name"].lower().startswith(cmdline.name_prefix):
                    self.log_debug("Skipped package {0}"
                                   .format(pkg["name"]))
                    continue
                arch = architectures.get(pkg["arch"], None)
                if not arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(pkg["arch"]))

                    continue

                repo_arch = architectures.get(repo_instance["arch"], None)
                if not repo_arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(repo_instance["arch"]))

                    continue

                build = (db.session.query(Build)
                         .filter(Build.base_package_name ==
                                 pkg["base_package_name"])
                         .filter(Build.version == pkg["version"])
                         .filter(Build.release == pkg["release"])
                         .filter(Build.epoch == pkg["epoch"])
                         .first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)
                    db.session.flush()

                build_arch = (db.session.query(BuildArch)
                         .filter(BuildArch.build_id == build.id)
                         .filter(BuildArch.arch_id == arch.id)
                         .first())

                if not build_arch:
                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                build_opsysrelease_arch = (
                       db.session.query(BuildOpSysReleaseArch)
                       .join(Build)
                       .join(OpSysRelease)
                       .join(Arch)
                       .filter(Build.id == build.id)
                       .filter(OpSys.name == repo_instance['opsys'])
                       .filter(OpSysRelease.version == repo_instance['release'])
                       .filter(Arch.name == repo_instance['arch'])
                       .first())

                if not build_opsysrelease_arch and repo_instance['release'] and repo_instance['opsys']:
                    self.log_info("Adding link between build {0}-{1} "
                            "operating system '{2}', release '{3} and "
                            "architecture {4}".format(pkg["base_package_name"],
                            pkg["version"], repo_instance['opsys'],
                            repo_instance['release'], repo_instance['arch']))

                    opsysrelease = (
                       db.session.query(OpSysRelease)
                       .filter(OpSys.name == repo_instance['opsys'])
                       .filter(OpSysRelease.version == repo_instance['release'])
                       .first())

                    bosra = BuildOpSysReleaseArch()
                    bosra.build = build
                    bosra.opsysrelease = opsysrelease
                    bosra.arch = repo_arch

                    db.session.add(bosra)
                    db.session.flush()

                package = (db.session.query(Package)
                           .filter(Package.name == pkg["name"])
                           .filter(Package.pkgtype == pkg["type"])
                           .filter(Package.build == build)
                           .filter(Package.arch == arch)
                           .first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attemps"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo.nogpgcheck)

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))
Beispiel #5
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if "$" in repo.url:  # parametrized
                self.log_info("Processing parametrized repo '{0}'"
                              .format(repo.name))

                if not repo.opsys_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an operating system")
                    return 1

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture")
                    return 1

                repo_instances += list(self._get_parametrized_variants(repo))
            else:
                repo_instance = repo_types[repo.type](repo.name, repo.url)
                repo_instances.append(repo_instance)

        cmdline.name_prefix = cmdline.name_prefix.lower()
        architectures = dict((x.name, x) for x in get_archs(db))

        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'"
                          .format(repo_instance.name, repo_instance.urls[0]))

            pkglist = repo_instance.list_packages(architectures.keys())
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}"
                               .format(num + 1, total, pkg["name"]))

                if not pkg["name"].lower().startswith(cmdline.name_prefix):
                    self.log_debug("Skipped package {0}"
                                   .format(pkg["name"]))
                    continue

                arch = architectures.get(pkg["arch"], None)
                if not arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(pkg["arch"]))

                    continue

                build = (db.session.query(Build)
                         .filter(Build.base_package_name ==
                                 pkg["base_package_name"])
                         .filter(Build.version == pkg["version"])
                         .filter(Build.release == pkg["release"])
                         .filter(Build.epoch == pkg["epoch"])
                         .first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)

                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                package = (db.session.query(Package)
                           .filter(Package.name == pkg["name"])
                           .filter(Package.pkgtype == pkg["type"])
                           .filter(Package.build == build)
                           .filter(Package.arch == arch)
                           .first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attemps"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo.nogpgcheck)

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))
Beispiel #6
0
    def restore_package_deps(self):
        print "Restoring package dependencies from rpms"
        for package in self.ses.query(Package):
            store_rpm_deps(self.db, package)

        self.ses.commit()
Beispiel #7
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if "$" in repo.url:  # parametrized
                self.log_info("Processing parametrized repo '{0}'"
                              .format(repo.name))

                if not repo.opsys_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an operating system")
                    return 1

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture")
                    return 1

                repo_instances += list(self._get_parametrized_variants(repo))
            else:
                repo_instance = repo_types[repo.type](repo.name, repo.url)
                repo_instances.append(repo_instance)

        architectures = map(lambda x: x.name, db.session.query(Arch))

        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'"
                          .format(repo_instance.name, repo_instance.urls[0]))

            pkglist = repo_instance.list_packages(architectures)
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}"
                               .format(num + 1, total, pkg["name"]))

                arch = get_arch_by_name(db, pkg["arch"])
                if not arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(pkg["arch"]))

                    continue

                build = (db.session.query(Build)
                         .filter(Build.base_package_name ==
                                 pkg["base_package_name"])
                         .filter(Build.version == pkg["version"])
                         .filter(Build.release == pkg["release"])
                         .filter(Build.epoch == pkg["epoch"])
                         .first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)

                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                package = (db.session.query(Package)
                           .filter(Package.name == pkg["name"])
                           .filter(Package.pkgtype == pkg["type"])
                           .filter(Package.build == build)
                           .filter(Package.arch == arch)
                           .first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attemps"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo.nogpgcheck)

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))