Exemplo n.º 1
0
    def assign_release_to_builds_testing(self, repo_type):
        # add repo
        self.assertEqual(self.call_action_ordered_args("repoadd", [
            "sample_repo", # NAME
            repo_type, # TYPE
            "file:///sample_rpms", # URL
        ]), 0)

        # add release
        self.assertEqual(self.call_action("releaseadd", {
            "opsys": "fedora",
            "opsys-release": "24",
            "status": "ACTIVE",
        }), 0)

        # add two builds
        build = Build()
        build.base_package_name = "build"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc24"
        self.db.session.add(build)

        build = Build()
        build.base_package_name = "build1"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)

        self.db.session.flush()

        systems['fedora'].get_released_builds = get_released_builds_mock

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(self.call_action_ordered_args(
            "assign-release-to-builds", [
            "Fedora", # OPSYS
            "24", # RELEASE
            "x86_64", # ARCH
            "--expression=fc24", # variant
        ]), 0)
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra + 1)

        self.assertEqual(self.call_action_ordered_args(
            "assign-release-to-builds", [
            "Fedora", # OPSYS
            "24", # RELEASE
            "x86_64", # ARCH
            "--released-builds", # variant
        ]), 0)
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra + 2)
Exemplo n.º 2
0
    def test_semver_compares_correctly(self):
        """
        Check whether semver comparison works correctly

        e.g. 3.0.11 > 3.0.1
        """

        ver_a = "3.0.11"
        build_a = Build()
        build_a.base_package_name = "sample"
        build_a.version = ver_a
        build_a.release = "1"
        build_a.epoch = "0"

        ver_b = "3.0.1"
        build_b = Build()
        build_b.base_package_name = "sample"
        build_b.version = ver_b
        build_b.release = "1"
        build_b.epoch = "0"

        ver_c = "3.0.0"
        build_c = Build()
        build_c.base_package_name = "sample"
        build_c.version = ver_c
        build_c.release = "1"
        build_c.epoch = "0"

        ver_d = "3.1.0"
        build_d = Build()
        build_d.base_package_name = "sample"
        build_d.version = ver_d
        build_d.release = "1"
        build_d.epoch = "0"

        self.db.session.add(build_a)
        self.db.session.add(build_b)
        self.db.session.add(build_c)
        self.db.session.add(build_d)
        self.db.session.flush()

        res = (self.db.session.query(Build.semver)
               .filter(Build.base_package_name == 'sample')
               .filter(Build.semver >= '3.0.1')
               .order_by(Build.semver).all())

        self.assertEqual(
            [item[0] for item in res], [ver_b, ver_a, ver_d])
Exemplo n.º 3
0
    def test_problem_json(self):
        problem = get_problem_by_id(self.db, 1)

        build = Build()
        build.base_package_name = "kernel"
        build.epoch = 0
        build.version = "3.12.10"
        build.release = "301.fc20"
        self.db.session.add(build)

        release = ProblemOpSysRelease()
        release.opsysrelease = get_releases(self.db, 'Fedora', '20').first()
        release.probable_fix_build = build
        release.problem = problem
        self.db.session.add(release)

        problem.reports[0].max_certainty = 99

        self.db.session.commit()

        response = self.app.get('/problems/%d/' % problem.id,
                                headers=([('Accept', 'application/json')]))

        self.assertEqual(response.mimetype, "application/json")

        data = json.loads(response.get_data(as_text=True))

        self.assertIn("solutions", data)
        self.assertEqual(len(data["solutions"]), 1)
        self.assertIn("note_text", data["solutions"][0])
        self.assertIn("%s-%s" % (build.version, build.release),
                      data["solutions"][0]["note_text"])
Exemplo n.º 4
0
    def test_problem_version_complex_filter(self):
        """
        Test if filtering yields two problems
        when only one of them matches release
        """

        build = Build()
        build.base_package_name = "kernel"
        build.epoch = 0
        build.version = "3.13.10"
        build.release = "100.fc20"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_x86_64
        pkg.name = "kernel"
        self.db.session.add(pkg)

        self.db.session.commit()

        self.save_report("ureport_kerneloops2")
        self.call_action("create-problems")

        self.db.session.commit()

        qs = "since_version=3.12&since_release=300.fc20"
        r = self.app.get("/problems/?{0}".format(qs))
        self.assertIn("problems/1", r.data)
        self.assertIn("problems/2", r.data)
Exemplo n.º 5
0
    def test_problem_version_complex_filter(self):
        """
        Test if filtering yields two problems
        when only one of them matches release
        """

        build = Build()
        build.base_package_name = "kernel"
        build.epoch = 0
        build.version = "3.13.10"
        build.release = "100.fc20"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_x86_64
        pkg.name = "kernel"
        self.db.session.add(pkg)

        self.db.session.commit()

        self.save_report("ureport_kerneloops2")
        self.call_action("create-problems")

        self.db.session.commit()

        qs = "since_version=3.12&since_release=300.fc20"
        r = self.app.get("/problems/?{0}".format(qs))
        self.assertIn(b"problems/1", r.data)
        self.assertIn(b"problems/2", r.data)
Exemplo n.º 6
0
    def test_store_rpm_deps(self):
        """
        """

        # add required stuff to db
        arch = Arch()
        arch.name = "noarch"
        self.db.session.add(arch)

        build = Build()
        build.base_package_name = "sample"
        build.version = "1"
        build.release = "1"
        build.epoch = "0"

        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build

        self.db.session.add(pkg)
        self.db.session.flush()

        # save sample rpm
        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg.save_lob("package", sample, truncate=True)

        # get dependencies
        res = store_rpm_deps(self.db, pkg, nogpgcheck=True)
        self.assertIs(res, True)

        expected_deps = [
            ("PROVIDES", "sample", 8),
            ("PROVIDES", "/sample", 0),
            ("PROVIDES", "nothing-new", 0),
            (
                "REQUIRES",
                "nothing",
                0,
            ),
            (
                "CONFLICTS",
                "surprisingly-nothing",
                0,
            ),
        ]

        found_deps = []

        for dep in self.db.session.query(PackageDependency).all():
            found_deps.append((dep.type, dep.name, dep.flags))

        for dep in expected_deps:
            self.assertIn(dep, found_deps)
Exemplo n.º 7
0
    def cleanup_unassigned_testing(self, repo_type):
        self.assign_release_to_builds_testing(repo_type)

        # add package and lob that will not be deleted
        pkg_stay = Package()
        pkg_stay.build = self.db.session.query(Build).first()
        pkg_stay.arch = self.db.session.query(Arch).first()
        pkg_stay.name = "pkg-test-stay"
        self.db.session.add(pkg_stay)
        self.db.session.flush()

        config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_stay.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_stay.has_lob("package"))

        # add build and package and lob that will be deleted
        build = Build()
        build.base_package_name = "build_unassigned"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)

        pkg_del = Package()
        pkg_del.build = build
        pkg_del.arch = self.db.session.query(Arch).first()
        pkg_del.name = "pkg-test-del"
        self.db.session.add(pkg_del)

        self.db.session.flush()

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_del.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_del.has_lob("package"))

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(self.call_action_ordered_args("cleanup-unassigned", [
            "--force"
        ]), 0)

        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra)

        self.assertFalse(pkg_del.has_lob("package"))
        self.assertTrue(pkg_stay.has_lob("package"))
Exemplo n.º 8
0
    def cleanup_unassigned_testing(self, repo_type):
        self.assign_release_to_builds_testing(repo_type)

        # add package and lob that will not be deleted
        pkg_stay = Package()
        pkg_stay.build = self.db.session.query(Build).first()
        pkg_stay.arch = self.db.session.query(Arch).first()
        pkg_stay.name = "pkg-test-stay"
        self.db.session.add(pkg_stay)
        self.db.session.flush()

        config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_stay.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_stay.has_lob("package"))

        # add build and package and lob that will be deleted
        build = Build()
        build.base_package_name = "build_unassigned"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)

        pkg_del = Package()
        pkg_del.build = build
        pkg_del.arch = self.db.session.query(Arch).first()
        pkg_del.name = "pkg-test-del"
        self.db.session.add(pkg_del)

        self.db.session.flush()

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_del.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_del.has_lob("package"))

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(
            self.call_action_ordered_args("cleanup-unassigned", ["--force"]),
            0)

        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra)

        self.assertFalse(pkg_del.has_lob("package"))
        self.assertTrue(pkg_stay.has_lob("package"))
Exemplo n.º 9
0
    def builds(self, count=10):
        self.begin("Builds")

        comps = self.ses.query(OpSysComponent).all()

        for comp in comps:
            for i in range(count):
                new = Build()
                new.component = comp
                new.epoch = 0
                new.version = "{0}.{1}".format(random.randrange(0, 5), random.randrange(0, 20))
                new.release = random.randrange(0, 100)
                self.add(new)

        self.commit()
Exemplo n.º 10
0
    def builds(self, count=10):
        self.begin('Builds')

        comps = self.ses.query(OpSysComponent).all()

        for comp in comps:
            for _ in range(count):
                new = Build()
                new.component = comp
                new.epoch = 0
                new.version = '{0}.{1}'.format(random.randrange(0, 5),
                                               random.randrange(0, 20))
                new.release = random.randrange(0, 100)
                self.add(new)

        self.commit()
Exemplo n.º 11
0
    def test_semver_stored_on_flush(self):
        """
        Make sure Build.semver is stored on session.flush
        """

        ver = "1.2.3-yeah666"
        build = Build()
        build.base_package_name = "sample"
        build.version = ver
        build.release = "1"
        build.epoch = "0"
        self.db.session.add(build)
        self.db.session.flush()

        self.assertEqual(build.version, build.semver)
        self.assertEqual(build.release, build.semrel)
Exemplo n.º 12
0
    def test_semver_stored_on_flush(self):
        """
        Make sure Build.semver is stored on session.flush
        """

        ver = "1.2.3-yeah666"
        build = Build()
        build.base_package_name = "sample"
        build.version = ver
        build.release = "1"
        build.epoch = "0"
        self.db.session.add(build)
        self.db.session.flush()

        self.assertEqual(build.version, build.semver)
        self.assertEqual(build.release, build.semrel)
Exemplo n.º 13
0
    def setUp(self):
        super(ProblemsTestCase, self).setUp()
        self.basic_fixtures()

        build = Build()
        build.base_package_name = "kernel"
        build.epoch = 0
        build.version = "3.12.10"
        build.release = "300.fc20"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_x86_64
        pkg.name = "kernel"
        self.db.session.add(pkg)

        self.db.session.commit()

        self.save_report("ureport_kerneloops")
        self.call_action("create-problems")

        self.db.session.commit()
Exemplo n.º 14
0
    def setUp(self):
        super(ProblemsTestCase, self).setUp()
        self.basic_fixtures()

        build = Build()
        build.base_package_name = "kernel"
        build.epoch = 0
        build.version = "3.12.10"
        build.release = "300.fc20"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_x86_64
        pkg.name = "kernel"
        self.db.session.add(pkg)

        self.db.session.commit()

        self.save_report("ureport_kerneloops")
        self.call_action("create-problems")

        self.db.session.commit()
Exemplo n.º 15
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if "$" in repo.url:  # parametrized
                self.log_info("Processing parametrized repo '{0}'"
                              .format(repo.name))

                if not repo.opsys_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an operating system")
                    return 1

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture")
                    return 1

                repo_instances += list(self._get_parametrized_variants(repo))
            else:
                repo_instance = repo_types[repo.type](repo.name, repo.url)
                repo_instances.append(repo_instance)

        cmdline.name_prefix = cmdline.name_prefix.lower()
        architectures = dict((x.name, x) for x in get_archs(db))

        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'"
                          .format(repo_instance.name, repo_instance.urls[0]))

            pkglist = repo_instance.list_packages(architectures.keys())
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}"
                               .format(num + 1, total, pkg["name"]))

                if not pkg["name"].lower().startswith(cmdline.name_prefix):
                    self.log_debug("Skipped package {0}"
                                   .format(pkg["name"]))
                    continue

                arch = architectures.get(pkg["arch"], None)
                if not arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(pkg["arch"]))

                    continue

                build = (db.session.query(Build)
                         .filter(Build.base_package_name ==
                                 pkg["base_package_name"])
                         .filter(Build.version == pkg["version"])
                         .filter(Build.release == pkg["release"])
                         .filter(Build.epoch == pkg["epoch"])
                         .first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)

                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                package = (db.session.query(Package)
                           .filter(Package.name == pkg["name"])
                           .filter(Package.pkgtype == pkg["type"])
                           .filter(Package.build == build)
                           .filter(Package.arch == arch)
                           .first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attemps"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo.nogpgcheck)

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))
Exemplo n.º 16
0
    def test_get_packages_and_their_reports_unknown_packages(self):
        """
        """

        # add required stuff to db
        arch = Arch()
        arch.name = "noarch"
        self.db.session.add(arch)

        arch1 = Arch()
        arch1.name = "x86_64"
        self.db.session.add(arch1)

        build = Build()
        build.base_package_name = "sample"
        build.version = "1"
        build.release = "1"
        build.epoch = "0"
        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build
        self.db.session.add(pkg)
        #print(pkg.nevra())

        #different arch
        pkg2 = Package()
        pkg2.name = "sample"
        pkg2.pkgtype = "rpm"
        pkg2.arch = arch1
        pkg2.build = build
        self.db.session.add(pkg2)
        #print(pkg2.nevra())

        build2 = Build()
        build2.base_package_name = "sample"
        build2.version = "1"
        build2.release = "2"
        build2.epoch = "0"
        self.db.session.add(build2)

        pkg3 = Package()
        pkg3.name = "sample"
        pkg3.pkgtype = "rpm"
        pkg3.arch = arch
        pkg3.build = build2
        self.db.session.add(pkg3)
        #print(pkg3.nevra())

        problem = Problem()
        self.db.session.add(problem)

        opsys = OpSys()
        opsys.name = "Fedora"
        self.db.session.add(opsys)

        opsys_component = OpSysComponent()
        opsys_component.name = "core"
        opsys_component.opsys = opsys
        self.db.session.add(opsys_component)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = opsys_component
        self.db.session.add(report)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = pkg.name
        report_unknown.epoch = pkg.build.epoch
        report_unknown.version = pkg.build.version
        report_unknown.release = pkg.build.release
        report_unknown.arch = pkg.arch
        report_unknown.count = 1
        self.db.session.add(report_unknown)

        report_unknown2 = ReportUnknownPackage()
        report_unknown2.report = report
        report_unknown2.name = pkg2.name
        report_unknown2.epoch = pkg2.build.epoch
        report_unknown2.version = pkg2.build.version
        report_unknown2.release = pkg2.build.release
        report_unknown2.arch = pkg2.arch
        report_unknown2.count = 1
        self.db.session.add(report_unknown2)

        report_unknown3 = ReportUnknownPackage()
        report_unknown3.report = report
        report_unknown3.name = "nonsense"
        report_unknown3.epoch = pkg.build.epoch
        report_unknown3.version = pkg.build.version
        report_unknown3.release = pkg.build.release
        report_unknown3.arch = pkg.arch
        report_unknown3.count = 1
        self.db.session.add(report_unknown3)

        self.db.session.flush()

        packages_and_their_reports_unknown_packages = \
            get_packages_and_their_reports_unknown_packages(self.db).all()
        self.assertEqual(len(packages_and_their_reports_unknown_packages), 2)
        self.assertIn(
            (pkg, report_unknown), packages_and_their_reports_unknown_packages)
        self.assertIn(
            (pkg2, report_unknown2), packages_and_their_reports_unknown_packages)
Exemplo n.º 17
0
    def test_store_rpm_provides(self):
        """
        """

        # add required stuff to db
        arch = Arch()
        arch.name = "noarch"
        self.db.session.add(arch)

        build = Build()
        build.base_package_name = "sample"
        build.version = "1"
        build.release = "1"
        build.epoch = "0"

        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build

        self.db.session.add(pkg)
        self.db.session.flush()

        # save sample rpm
        with open("sample_rpms/sample-1.0-1.fc18.noarch.rpm",
                  mode='rb') as sample:
            pkg.save_lob("package", sample, truncate=True)

        # get dependencies
        store_rpm_provides(self.db, pkg, nogpgcheck=True)

        expected_deps = [
            ("PROVIDES", "sample", 8),
            ("PROVIDES", "/sample", 0),
            ("PROVIDES", "nothing-new", 0),
        ]

        found_deps = []

        for dep in self.db.session.query(PackageDependency).all():
            found_deps.append((dep.type, dep.name, dep.flags))

        self.assertCountEqual(found_deps, expected_deps)

        build = Build()
        build.base_package_name = "sample-broken"
        build.version = "1"
        build.release = "1"
        build.epoch = 0

        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample-broken"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build

        self.db.session.add(pkg)
        self.db.session.flush()

        with open("sample_rpms/sample-broken-1-1.fc22.noarch.rpm",
                  mode='rb') as rpm_file:
            pkg.save_lob("package", rpm_file, truncate=True)

        with self.assertLogs(level=logging.WARNING) as captured_logs:
            store_rpm_provides(self.db, pkg, nogpgcheck=True)

        self.assertEqual(captured_logs.output, [
            "WARNING:faf.pyfaf.faf_rpm:Unparsable EVR ‘%{epoch}:1’ of "
            "zabbix in Provides of sample-broken: EVR string "
            "contains a non-numeric epoch: %{epoch}. Skipping"
        ])

        # Safety flush
        self.db.session.flush()

        query = self.db.session.query(PackageDependency)
        # Filter out rich RPM dependencies
        dependencies = filter(lambda x: "rpmlib" not in x.name, query.all())

        # Only provides that were correctly formatted were added
        expected_deps.extend([
            ("PROVIDES", "sample-broken", 8),
            ("PROVIDES", "happiness", 0),
            ("PROVIDES", "joy", 0),
            ("PROVIDES", "love", 0),
        ])

        self.assertCountEqual(
            ((x.type, x.name, x.flags) for x in dependencies), expected_deps)

        build = Build()
        build.base_package_name = "sample-provides-too-long"
        build.version = "1"
        build.release = "1"
        build.epoch = 0

        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample-provides-too-long"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build

        self.db.session.add(pkg)
        self.db.session.flush()

        with open("sample_rpms/sample-provides-too-long-1-1.fc33.noarch.rpm",
                  mode='rb') as rpm_file:
            pkg.save_lob("package", rpm_file, truncate=True)

        with self.assertLogs(level=logging.WARNING) as captured_logs:
            store_rpm_provides(self.db, pkg, nogpgcheck=True)

        self.assertEqual(captured_logs.output, [
            "WARNING:faf.pyfaf.faf_rpm:Provides item in RPM header of "
            "sample-provides-too-long longer than 1024 characters. "
            "Skipping"
        ])

        # Safety flush
        self.db.session.flush()

        query = self.db.session.query(PackageDependency)
        # Filter out rich RPM dependencies
        dependencies = filter(lambda x: "rpmlib" not in x.name, query.all())

        # Only provides that were correctly formatted were added
        expected_deps.extend([
            ("PROVIDES", "sample-provides-too-long", 8),
            ("PROVIDES", "one-thing", 0),
            ("PROVIDES", "another-thing", 0),
            ("PROVIDES", "penultimate-item", 0),
            ("PROVIDES", "the-last-one", 0),
        ])

        self.assertCountEqual(
            ((x.type, x.name, x.flags) for x in dependencies), expected_deps)
Exemplo n.º 18
0
    def assign_release_to_builds_testing(self, repo_type):
        # add repo
        self.assertEqual(
            self.call_action_ordered_args(
                "repoadd",
                [
                    "sample_repo",  # NAME
                    repo_type,  # TYPE
                    "file:///sample_rpms",  # URL
                ]),
            0)

        # add release
        self.assertEqual(
            self.call_action("releaseadd", {
                "opsys": "fedora",
                "opsys-release": "24",
                "status": "ACTIVE",
            }), 0)

        # add two builds
        build = Build()
        build.base_package_name = "build"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc24"
        self.db.session.add(build)
        self.db.session.flush()

        build_arch = BuildArch(arch_id=self.arch_x86_64.id, build_id=build.id)
        self.db.session.add(build_arch)
        self.db.session.flush()

        build = Build()
        build.base_package_name = "build1"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)
        self.db.session.flush()

        build_arch = BuildArch(arch_id=self.arch_x86_64.id, build_id=build.id)

        self.db.session.add(build_arch)
        self.db.session.flush()

        systems['fedora'].get_released_builds = get_released_builds_mock

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(
            self.call_action_ordered_args(
                "assign-release-to-builds",
                [
                    "Fedora",  # OPSYS
                    "24",  # RELEASE
                    "x86_64",  # ARCH
                    "--expression=fc24",  # variant
                ]),
            0)
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra + 1)

        self.assertEqual(
            self.call_action_ordered_args(
                "assign-release-to-builds",
                [
                    "Fedora",  # OPSYS
                    "24",  # RELEASE
                    "x86_64",  # ARCH
                    "--released-builds",  # variant
                ]),
            0)
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra + 2)
Exemplo n.º 19
0
    def test_get_packages_and_their_reports_unknown_packages(self):
        """
        """

        # add required stuff to db
        arch = Arch()
        arch.name = "noarch"
        self.db.session.add(arch)

        arch1 = Arch()
        arch1.name = "x86_64"
        self.db.session.add(arch1)

        build = Build()
        build.base_package_name = "sample"
        build.version = "1"
        build.release = "1"
        build.epoch = "0"
        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build
        self.db.session.add(pkg)
        #print(pkg.nevra())

        #different arch
        pkg2 = Package()
        pkg2.name = "sample"
        pkg2.pkgtype = "rpm"
        pkg2.arch = arch1
        pkg2.build = build
        self.db.session.add(pkg2)
        #print(pkg2.nevra())

        build2 = Build()
        build2.base_package_name = "sample"
        build2.version = "1"
        build2.release = "2"
        build2.epoch = "0"
        self.db.session.add(build2)

        pkg3 = Package()
        pkg3.name = "sample"
        pkg3.pkgtype = "rpm"
        pkg3.arch = arch
        pkg3.build = build2
        self.db.session.add(pkg3)
        #print(pkg3.nevra())

        problem = Problem()
        self.db.session.add(problem)

        opsys = OpSys()
        opsys.name = "Fedora"
        self.db.session.add(opsys)

        opsys_component = OpSysComponent()
        opsys_component.name = "core"
        opsys_component.opsys = opsys
        self.db.session.add(opsys_component)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = opsys_component
        self.db.session.add(report)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = pkg.name
        report_unknown.epoch = pkg.build.epoch
        report_unknown.version = pkg.build.version
        report_unknown.release = pkg.build.release
        report_unknown.arch = pkg.arch
        report_unknown.count = 1
        self.db.session.add(report_unknown)

        report_unknown2 = ReportUnknownPackage()
        report_unknown2.report = report
        report_unknown2.name = pkg2.name
        report_unknown2.epoch = pkg2.build.epoch
        report_unknown2.version = pkg2.build.version
        report_unknown2.release = pkg2.build.release
        report_unknown2.arch = pkg2.arch
        report_unknown2.count = 1
        self.db.session.add(report_unknown2)

        report_unknown3 = ReportUnknownPackage()
        report_unknown3.report = report
        report_unknown3.name = "nonsense"
        report_unknown3.epoch = pkg.build.epoch
        report_unknown3.version = pkg.build.version
        report_unknown3.release = pkg.build.release
        report_unknown3.arch = pkg.arch
        report_unknown3.count = 1
        self.db.session.add(report_unknown3)

        self.db.session.flush()

        packages_and_their_reports_unknown_packages = \
            get_packages_and_their_reports_unknown_packages(self.db).all()
        self.assertEqual(len(packages_and_their_reports_unknown_packages), 2)
        self.assertIn((pkg, report_unknown),
                      packages_and_their_reports_unknown_packages)
        self.assertIn((pkg2, report_unknown2),
                      packages_and_their_reports_unknown_packages)
Exemplo n.º 20
0
    def run(self, cmdline, db) -> None:
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.REPO and repo.name not in cmdline.REPO:
                continue

            if cmdline.match_repos not in repo.name:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if repo.opsys_list:  # parametrized
                self.log_info("Processing parametrized repo '{0}'".format(
                    repo.name))

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture, skipping")
                    continue
                try:
                    repo_instances += list(
                        self._get_parametrized_variants(repo))
                except:  # pylint: disable=bare-except
                    self.log_error(
                        "No valid mirror for repository '{0}', skipping".
                        format(repo.name))
                    continue

            elif repo.opsysrelease_list:
                self.log_info(
                    "Processing repo '{0}' assigned with OpSysRelease".format(
                        repo.name))

                if not repo.arch_list:
                    self.log_error("OpSysRelease repository is not assigned"
                                   " with an architecture, skipping")
                    continue
                try:
                    repo_instances += list(
                        self._get_opsysrelease_variants(repo))
                except:  # pylint: disable=bare-except
                    self.log_error(
                        "No valid mirror for repository '{0}', skipping".
                        format(repo.name))
                    continue

            else:
                if any('$' in url.url for url in repo.url_list):
                    self.log_error("No operating system assigned to"
                                   "parametrized repo '{0}', skipping".format(
                                       repo.name))
                    continue
                for arch in repo.arch_list:
                    try:
                        repo_instance = {
                            'instance':
                            repo_types[repo.type](
                                repo.name, [url.url for url in repo.url_list]),
                            'opsys':
                            None,
                            'release':
                            None,
                            'arch':
                            arch.name,
                            'nogpgcheck':
                            repo.nogpgcheck
                        }
                        repo_instances.append(repo_instance)
                    except:  # pylint: disable=bare-except
                        self.log_error(
                            "No valid mirror for repository '{0}', skipping".
                            format(repo.name))
                        continue

        cmdline.name_prefix = cmdline.name_prefix.lower()
        architectures = dict((x.name, x) for x in get_archs(db))
        for repo_instance in repo_instances:
            self.log_info(
                "Processing repository '{0}' assigned to OS release '{1}' and arch '{2}', URL: '{3}'"
                .format(repo_instance['instance'].name,
                        repo_instance['release'], repo_instance['arch'],
                        repo_instance['instance'].urls))

            if cmdline.no_cache:
                repo_instance['instance'].cache_lifetime = 0

            pkglist = \
                repo_instance['instance'].list_packages(list(architectures.keys()))
            total = len(pkglist)

            self.log_info("Repository has {0} packages".format(total))

            for num, pkg in enumerate(pkglist, start=1):
                self.log_info("[%d / %d] Processing package %s", num, total,
                              pkg["name"])

                if not pkg["name"].lower().startswith(cmdline.name_prefix):
                    self.log_debug("Skipped package %s", pkg["name"])
                    continue
                arch = architectures.get(pkg["arch"], None)
                if not arch:
                    self.log_error(
                        "Architecture '{0}' not found, skipping".format(
                            pkg["arch"]))

                    continue

                repo_arch = architectures.get(repo_instance["arch"], None)
                if not repo_arch:
                    self.log_error(
                        "Architecture '{0}' not found, skipping".format(
                            repo_instance["arch"]))

                    continue

                build = (db.session.query(Build).filter(
                    Build.base_package_name == pkg["base_package_name"]).
                         filter(Build.version == pkg["version"]).filter(
                             Build.release == pkg["release"]).filter(
                                 Build.epoch == pkg["epoch"]).first())

                if not build:
                    self.log_debug("Adding build %s-%s",
                                   pkg["base_package_name"], pkg["version"])

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)
                    db.session.flush()

                build_arch = (db.session.query(BuildArch).filter(
                    BuildArch.build_id == build.id).filter(
                        BuildArch.arch_id == arch.id).first())

                if not build_arch:
                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                build_opsysrelease_arch = (db.session.query(
                    BuildOpSysReleaseArch
                ).join(Build).join(OpSysRelease).join(Arch).filter(
                    Build.id == build.id).filter(
                        OpSys.name == repo_instance['opsys']).filter(
                            OpSysRelease.version == repo_instance['release']
                        ).filter(Arch.name == repo_instance['arch']).first())

                if not build_opsysrelease_arch and repo_instance[
                        'release'] and repo_instance['opsys']:
                    self.log_info("Adding link between build '{0}-{1}' "
                                  "and operating system '{2} {3} {4}'".format(
                                      pkg["base_package_name"], pkg["version"],
                                      repo_instance['opsys'],
                                      repo_instance['release'],
                                      repo_instance['arch']))

                    opsysrelease = (db.session.query(OpSysRelease).filter(
                        OpSys.name == repo_instance['opsys']).filter(
                            OpSysRelease.version ==
                            repo_instance['release']).first())

                    bosra = BuildOpSysReleaseArch()
                    bosra.build = build
                    bosra.opsysrelease = opsysrelease
                    bosra.arch = repo_arch

                    db.session.add(bosra)
                    db.session.flush()

                package = (db.session.query(Package).filter(
                    Package.name == pkg["name"]).filter(
                        Package.pkgtype == pkg["type"]).filter(
                            Package.build == build).filter(
                                Package.arch == arch).first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error(
                            "Exception ({0}) after multiple attempts"
                            " while trying to download {1},"
                            " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    if pkg["type"] == "rpm":
                        try:
                            store_rpm_provides(db, package,
                                               repo_instance['nogpgcheck'])
                        except FafError as ex:
                            self.log_error(
                                "Post-processing failed, skipping: {}".format(
                                    ex))
                            db.session.delete(package)
                            db.session.flush()
                            continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:  # pylint: disable=broad-except
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package %s", pkg["filename"])
                    if not package.has_lob(
                            "package") and not cmdline.no_download_rpm:
                        self.log_info(
                            "Package {} does not have a LOB. Re-downloading.".
                            format(pkg["name"]))
                        try:
                            self._download(package, "package", pkg["url"])
                        except (FafError, URLError) as exc:
                            self.log_error(
                                "Exception ({0}) after multiple attempts"
                                " while trying to download {1},"
                                " skipping.".format(exc, pkg["url"]))
Exemplo n.º 21
0
    def releasedel_testing(self, repo_type):
        self.assertEqual(
            self.call_action("releasedel", {
                "opsys": "fedora",
                "opsys-release": "1337",
            }), 1)

        # add release f24 and assign two builds
        self.assign_release_to_builds_testing(repo_type)

        # assign sample_repo to f24 release (sample_repo comes from ^^^)
        self.assertEqual(
            self.call_action_ordered_args(
                "repoassign",
                [
                    "sample_repo",  # NAME
                    "Fedora 24",  # OPSYS
                    "x86_64",  # ARCH
                ]),
            0)

        tracker = Bugtracker(name="fedora-bugzilla")
        self.db.session.add(tracker)

        self.save_report('ureport_f20')
        self.save_report('ureport_core')
        self.call_action("create-problems")

        bzuser = BzUser(name="Fake user",
                        email="*****@*****.**",
                        real_name="Fake name",
                        can_login=False)

        self.db.session.add(bzuser)

        bug = BzBug()
        bug.id = 123456
        bug.summary = "Fake bug"
        bug.status = "NEW"
        bug.creation_time = datetime.now()
        bug.last_change_time = datetime.now()
        bug.whiteboard = "empty"
        bug.tracker = tracker
        bug.creator = bzuser
        bug.component = self.comp_faf
        bug.opsysrelease = self.release_20
        bug.private = False

        self.db.session.add(bug)

        # add package and lob that will be deleted
        pkg_del = Package()
        pkg_del.build = self.db.session.query(Build).first()
        pkg_del.arch = self.db.session.query(Arch).first()
        pkg_del.name = "pkg-test-del"
        self.db.session.add(pkg_del)
        self.db.session.flush()

        config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_del.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_del.has_lob("package"))

        # add build and package and lob that will not be deleted
        build = Build()
        build.base_package_name = "build_unassigned"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)

        pkg_stay = Package()
        pkg_stay.build = build
        pkg_stay.arch = self.db.session.query(Arch).first()
        pkg_stay.name = "pkg-test-stay"
        self.db.session.add(pkg_stay)
        self.db.session.flush()

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_stay.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_stay.has_lob("package"))

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()

        # delete release f24 with assigned repos and builds with lobs
        self.assertEqual(
            self.call_action("releasedel", {
                "opsys": "fedora",
                "opsys-release": "24",
            }), 0)

        self.assertEqual(bosra, init_bosra)
        self.assertTrue(pkg_stay.has_lob("package"))
        self.assertFalse(pkg_del.has_lob("package"))

        self.assertEqual(self.db.session.query(Report).count(), 2)
        self.assertEqual(self.db.session.query(Problem).count(), 2)

        # delete release f20 with assigned bugs, reports and problems
        self.assertEqual(
            self.call_action("releasedel", {
                "opsys": "fedora",
                "opsys-release": "20",
            }), 0)

        self.assertEqual(self.db.session.query(Report).count(), 0)
        self.assertEqual(self.db.session.query(Problem).count(), 0)

        # attempt to delete deleted release f20
        self.assertEqual(
            self.call_action("releasedel", {
                "opsys": "fedora",
                "opsys-release": "20",
            }), 1)
Exemplo n.º 22
0
    def setUp(self):
        super(MPFTestCase, self).setUp()
        self.basic_fixtures()

        # Fixed problem 1
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_systemd
        self.db.session.add(report)

        with open("sample_reports/ureport_systemd2", "r") as file:
            self.ureport_systemd2 = json.load(file)
        with open("sample_reports/ureport_systemd77", "r") as file:
            self.ureport_systemd77 = json.load(file)
        problemplugin = problemtypes[self.ureport_systemd2["problem"]["type"]]
        report_hash = problemplugin.hash_ureport(
            self.ureport_systemd2["problem"])
        db_hash = ReportHash()
        db_hash.hash = report_hash
        db_hash.report = report
        self.db.session.add(db_hash)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = "systemd-test"
        report_unknown.epoch = 0
        report_unknown.version = 2
        report_unknown.release = 1
        report_unknown.semver = "2.0.0"
        report_unknown.semrel = "1.0.0"
        report_unknown.arch = self.arch_i686
        report_unknown.count = 1
        report_unknown.type = "CRASHED"
        self.db.session.add(report_unknown)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = "faf"
        report_unknown.epoch = 0
        report_unknown.version = 1
        report_unknown.release = 1
        report_unknown.semver = "1.0.0"
        report_unknown.semrel = "1.0.0"
        report_unknown.arch = self.arch_i686
        report_unknown.count = 1
        report_unknown.type = "RELATED"
        self.db.session.add(report_unknown)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_19
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_fixed1 = problem

        # Fixed problem 2
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_faf
        self.db.session.add(report)

        build = Build()
        build.base_package_name = "faf"
        build.epoch = 0
        build.version = "3.4"
        build.release = "5"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_i686
        pkg.name = "faf-test"
        self.db.session.add(pkg)

        rpkg = ReportPackage()
        rpkg.report = report
        rpkg.type = "CRASHED"
        rpkg.installed_package = pkg
        rpkg.count = 66
        self.db.session.add(rpkg)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_fixed2 = problem

        # Unfixed problem 1
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_systemd
        self.db.session.add(report)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = "systemd"
        report_unknown.epoch = 0
        report_unknown.version = 3
        report_unknown.release = 1
        report_unknown.semver = "3.0.0"
        report_unknown.semrel = "1.0.0"
        report_unknown.arch = self.arch_i686
        report_unknown.count = 1
        report_unknown.type = "CRASHED"
        self.db.session.add(report_unknown)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_unfixed1 = problem

        # Unfixed problem 2
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_systemd
        self.db.session.add(report)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = "systemd"
        report_unknown.epoch = 0
        report_unknown.version = 3
        report_unknown.release = 0
        report_unknown.semver = "3.0.0"
        report_unknown.semrel = "0.0.0"
        report_unknown.arch = self.arch_i686
        report_unknown.count = 1
        report_unknown.type = "CRASHED"
        self.db.session.add(report_unknown)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_unfixed2 = problem

        # Unfixed problem 3
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_faf
        self.db.session.add(report)

        build = Build()
        build.base_package_name = "faf"
        build.epoch = 0
        build.version = "3.110"
        build.release = "5"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_i686
        pkg.name = "faf-test"
        self.db.session.add(pkg)

        rpkg = ReportPackage()
        rpkg.report = report
        rpkg.type = "CRASHED"
        rpkg.installed_package = pkg
        rpkg.count = 66
        self.db.session.add(rpkg)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_unfixed3 = problem

        self.db.session.flush()

        systems['fedora'].get_released_builds = get_released_builds_mock
        systems['fedora'].build_aging_days = 7
Exemplo n.º 23
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if "$" in repo.url:  # parametrized
                self.log_info("Processing parametrized repo '{0}'"
                              .format(repo.name))

                if not repo.opsys_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an operating system")
                    return 1

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture")
                    return 1

                repo_instances += list(self._get_parametrized_variants(repo))
            else:
                repo_instance = repo_types[repo.type](repo.name, repo.url)
                repo_instances.append(repo_instance)

        architectures = map(lambda x: x.name, db.session.query(Arch))

        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'"
                          .format(repo_instance.name, repo_instance.urls[0]))

            pkglist = repo_instance.list_packages(architectures)
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}"
                               .format(num + 1, total, pkg["name"]))

                arch = get_arch_by_name(db, pkg["arch"])
                if not arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(pkg["arch"]))

                    continue

                build = (db.session.query(Build)
                         .filter(Build.base_package_name ==
                                 pkg["base_package_name"])
                         .filter(Build.version == pkg["version"])
                         .filter(Build.release == pkg["release"])
                         .filter(Build.epoch == pkg["epoch"])
                         .first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)

                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                package = (db.session.query(Package)
                           .filter(Package.name == pkg["name"])
                           .filter(Package.pkgtype == pkg["type"])
                           .filter(Package.build == build)
                           .filter(Package.arch == arch)
                           .first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attemps"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo.nogpgcheck)

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))
Exemplo n.º 24
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if repo.opsys_list:  # parametrized
                self.log_info("Processing parametrized repo '{0}'"
                              .format(repo.name))

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture")
                    return 1
                try:
                    repo_instances += list(self._get_parametrized_variants(repo))
                except:
                    self.log_error("No valid mirror for repository {0}"
                                                    .format(repo.name))
                    return 1


            elif repo.opsysrelease_list:
                self.log_info("Processing repo '{0}' assigned with OpSysRelease"
                          .format(repo.name))

                if not repo.arch_list:
                    self.log_error("OpSysRelease repository is not assigned"
                                   " with an architecture")
                    return 1
                try:
                    repo_instances += list(self._get_opsysrelease_variants(repo))
                except:
                    self.log_error("No valid mirror for repository {0}"
                                                    .format(repo.name))
                    return 1

            else:
                if any('$' in url.url for url in repo.url_list):
                    self.log_error("No operating system assigned to"
                            "parametrized repo '{0}".format(repo.name))
                    return 1
                for arch in repo.arch_list:
                    try:
                        repo_instance = {
                                'instance' : repo_types[repo.type](repo.name,
                                            [url.url for url in repo.url_list]),
                                'opsys' : None,
                                'release' : None,
                                'arch' : arch.name}
                        repo_instances.append(repo_instance)
                    except:
                        self.log_error("No valid mirror for repository {0}"
                                                        .format(repo.name))
                        return 1

        cmdline.name_prefix = cmdline.name_prefix.lower()
        architectures = dict((x.name, x) for x in get_archs(db))
        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'"
                          .format(repo_instance['instance'].name,
                              repo_instance['instance'].urls))

            pkglist = \
                repo_instance['instance'].list_packages(architectures.keys())
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}"
                               .format(num + 1, total, pkg["name"]))

                if not pkg["name"].lower().startswith(cmdline.name_prefix):
                    self.log_debug("Skipped package {0}"
                                   .format(pkg["name"]))
                    continue
                arch = architectures.get(pkg["arch"], None)
                if not arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(pkg["arch"]))

                    continue

                repo_arch = architectures.get(repo_instance["arch"], None)
                if not repo_arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(repo_instance["arch"]))

                    continue

                build = (db.session.query(Build)
                         .filter(Build.base_package_name ==
                                 pkg["base_package_name"])
                         .filter(Build.version == pkg["version"])
                         .filter(Build.release == pkg["release"])
                         .filter(Build.epoch == pkg["epoch"])
                         .first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)
                    db.session.flush()

                build_arch = (db.session.query(BuildArch)
                         .filter(BuildArch.build_id == build.id)
                         .filter(BuildArch.arch_id == arch.id)
                         .first())

                if not build_arch:
                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                build_opsysrelease_arch = (
                       db.session.query(BuildOpSysReleaseArch)
                       .join(Build)
                       .join(OpSysRelease)
                       .join(Arch)
                       .filter(Build.id == build.id)
                       .filter(OpSys.name == repo_instance['opsys'])
                       .filter(OpSysRelease.version == repo_instance['release'])
                       .filter(Arch.name == repo_instance['arch'])
                       .first())

                if not build_opsysrelease_arch and repo_instance['release'] and repo_instance['opsys']:
                    self.log_info("Adding link between build {0}-{1} "
                            "operating system '{2}', release '{3} and "
                            "architecture {4}".format(pkg["base_package_name"],
                            pkg["version"], repo_instance['opsys'],
                            repo_instance['release'], repo_instance['arch']))

                    opsysrelease = (
                       db.session.query(OpSysRelease)
                       .filter(OpSys.name == repo_instance['opsys'])
                       .filter(OpSysRelease.version == repo_instance['release'])
                       .first())

                    bosra = BuildOpSysReleaseArch()
                    bosra.build = build
                    bosra.opsysrelease = opsysrelease
                    bosra.arch = repo_arch

                    db.session.add(bosra)
                    db.session.flush()

                package = (db.session.query(Package)
                           .filter(Package.name == pkg["name"])
                           .filter(Package.pkgtype == pkg["type"])
                           .filter(Package.build == build)
                           .filter(Package.arch == arch)
                           .first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attemps"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo.nogpgcheck)

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))
Exemplo n.º 25
0
    def test_store_rpm_deps(self):
        """
        """

        # add required stuff to db
        arch = Arch()
        arch.name = "noarch"
        self.db.session.add(arch)

        build = Build()
        build.base_package_name = "sample"
        build.version = "1"
        build.release = "1"
        build.epoch = "0"

        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build

        self.db.session.add(pkg)
        self.db.session.flush()

        # save sample rpm
        with open("sample_rpms/sample-1.0-1.fc18.noarch.rpm",
                  mode='rb') as sample:
            pkg.save_lob("package", sample, truncate=True)

        # get dependencies
        store_rpm_deps(self.db, pkg, nogpgcheck=True)

        expected_deps = [
            ("PROVIDES", "sample", 8),
            ("PROVIDES", "/sample", 0),
            ("PROVIDES", "nothing-new", 0),
            (
                "REQUIRES",
                "nothing",
                0,
            ),
            (
                "CONFLICTS",
                "surprisingly-nothing",
                0,
            ),
        ]

        found_deps = []

        for dep in self.db.session.query(PackageDependency).all():
            found_deps.append((dep.type, dep.name, dep.flags))

        for dep in expected_deps:
            self.assertIn(dep, found_deps)

        build = Build()
        build.base_package_name = "sample-broken"
        build.version = "1"
        build.release = "1"
        build.epoch = 0

        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample-broken"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build

        self.db.session.add(pkg)
        self.db.session.flush()

        with open("sample_rpms/sample-broken-1-1.el7.noarch.rpm",
                  mode='rb') as sample:
            pkg.save_lob("package", sample, truncate=True)

        with self.assertRaises(FafError):
            store_rpm_deps(self.db, pkg, nogpgcheck=True)

        # Safety flush
        self.db.session.flush()

        query = self.db.session.query(PackageDependency)
        # Filter out rich RPM dependencies
        dependencies = filter(lambda x: "rpmlib" not in x.name, query.all())

        # The transaction was rolled back and nothing new was added
        self.assertCountEqual(
            map(lambda x: (x.type, x.name, x.flags), dependencies),
            expected_deps)
Exemplo n.º 26
0
    def releasedel_testing(self, repo_type):
        self.assertEqual(self.call_action("releasedel"), 1)
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "FooBarOS",
        }), 1)
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "fedora",
            "opsys-release": "1337",
        }), 1)

        # add release f24 and assign two builds
        self.assign_release_to_builds_testing(repo_type)

        # assign sample_repo to f24 release (sample_repo comes from ^^^)
        self.assertEqual(self.call_action_ordered_args("repoassign", [
            "sample_repo",  # NAME
            "Fedora 24",  # OPSYS
            "x86_64",  # ARCH
        ]), 0)

        tracker = Bugtracker(name="fedora-bugzilla")
        self.db.session.add(tracker)

        self.save_report('ureport_f20')
        self.save_report('ureport_core')
        self.call_action("create-problems")

        bzuser = BzUser(name="Fake user",
                        email="*****@*****.**",
                        real_name="Fake name",
                        can_login=False)

        self.db.session.add(bzuser)

        bug = BzBug()
        bug.id = 123456
        bug.summary = "Fake bug"
        bug.status = "NEW"
        bug.creation_time = datetime.now()
        bug.last_change_time = datetime.now()
        bug.whiteboard = "empty"
        bug.tracker = tracker
        bug.creator = bzuser
        bug.component = self.comp_faf
        bug.opsysrelease = self.release_20
        bug.private = False

        self.db.session.add(bug)

        # add package and lob that will be deleted
        pkg_del = Package()
        pkg_del.build = self.db.session.query(Build).first()
        pkg_del.arch = self.db.session.query(Arch).first()
        pkg_del.name = "pkg-test-del"
        self.db.session.add(pkg_del)
        self.db.session.flush()

        config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_del.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_del.has_lob("package"))

        # add build and package and lob that will not be deleted
        build = Build()
        build.base_package_name = "build_unassigned"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)

        pkg_stay = Package()
        pkg_stay.build = build
        pkg_stay.arch = self.db.session.query(Arch).first()
        pkg_stay.name = "pkg-test-stay"
        self.db.session.add(pkg_stay)
        self.db.session.flush()

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_stay.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_stay.has_lob("package"))

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()

        # delete release f24 with assigned repos and builds with lobs
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "fedora",
            "opsys-release": "24",
        }), 0)

        self.assertEqual(bosra, init_bosra)
        self.assertTrue(pkg_stay.has_lob("package"))
        self.assertFalse(pkg_del.has_lob("package"))

        self.assertEqual(self.db.session.query(Report).count(), 2)
        self.assertEqual(self.db.session.query(Problem).count(), 2)

        # delete release f20 with assigned bugs, reports and problems
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "fedora",
            "opsys-release": "20",
        }), 0)

        self.assertEqual(self.db.session.query(Report).count(), 0)
        self.assertEqual(self.db.session.query(Problem).count(), 0)

        # attempt to delete deleted release f20
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "fedora",
            "opsys-release": "20",
        }), 1)