Esempio n. 1
0
    def assign_release_to_builds_testing(self, repo_type):
        # add repo
        self.assertEqual(self.call_action_ordered_args("repoadd", [
            "sample_repo", # NAME
            repo_type, # TYPE
            "file:///sample_rpms", # URL
        ]), 0)

        # add release
        self.assertEqual(self.call_action("releaseadd", {
            "opsys": "fedora",
            "opsys-release": "24",
            "status": "ACTIVE",
        }), 0)

        # add two builds
        build = Build()
        build.base_package_name = "build"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc24"
        self.db.session.add(build)

        build = Build()
        build.base_package_name = "build1"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)

        self.db.session.flush()

        systems['fedora'].get_released_builds = get_released_builds_mock

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(self.call_action_ordered_args(
            "assign-release-to-builds", [
            "Fedora", # OPSYS
            "24", # RELEASE
            "x86_64", # ARCH
            "--expression=fc24", # variant
        ]), 0)
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra + 1)

        self.assertEqual(self.call_action_ordered_args(
            "assign-release-to-builds", [
            "Fedora", # OPSYS
            "24", # RELEASE
            "x86_64", # ARCH
            "--released-builds", # variant
        ]), 0)
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra + 2)
Esempio n. 2
0
    def test_semver_compares_correctly(self):
        """
        Check whether semver comparison works correctly

        e.g. 3.0.11 > 3.0.1
        """

        ver_a = "3.0.11"
        build_a = Build()
        build_a.base_package_name = "sample"
        build_a.version = ver_a
        build_a.release = "1"
        build_a.epoch = "0"

        ver_b = "3.0.1"
        build_b = Build()
        build_b.base_package_name = "sample"
        build_b.version = ver_b
        build_b.release = "1"
        build_b.epoch = "0"

        ver_c = "3.0.0"
        build_c = Build()
        build_c.base_package_name = "sample"
        build_c.version = ver_c
        build_c.release = "1"
        build_c.epoch = "0"

        ver_d = "3.1.0"
        build_d = Build()
        build_d.base_package_name = "sample"
        build_d.version = ver_d
        build_d.release = "1"
        build_d.epoch = "0"

        self.db.session.add(build_a)
        self.db.session.add(build_b)
        self.db.session.add(build_c)
        self.db.session.add(build_d)
        self.db.session.flush()

        res = (self.db.session.query(Build.semver)
               .filter(Build.base_package_name == 'sample')
               .filter(Build.semver >= '3.0.1')
               .order_by(Build.semver).all())

        self.assertEqual(
            [item[0] for item in res], [ver_b, ver_a, ver_d])
Esempio n. 3
0
    def test_problem_version_complex_filter(self):
        """
        Test if filtering yields two problems
        when only one of them matches release
        """

        build = Build()
        build.base_package_name = "kernel"
        build.epoch = 0
        build.version = "3.13.10"
        build.release = "100.fc20"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_x86_64
        pkg.name = "kernel"
        self.db.session.add(pkg)

        self.db.session.commit()

        self.save_report("ureport_kerneloops2")
        self.call_action("create-problems")

        self.db.session.commit()

        qs = "since_version=3.12&since_release=300.fc20"
        r = self.app.get("/problems/?{0}".format(qs))
        self.assertIn(b"problems/1", r.data)
        self.assertIn(b"problems/2", r.data)
Esempio n. 4
0
    def cleanup_unassigned_testing(self, repo_type):
        self.assign_release_to_builds_testing(repo_type)

        # add package and lob that will not be deleted
        pkg_stay = Package()
        pkg_stay.build = self.db.session.query(Build).first()
        pkg_stay.arch = self.db.session.query(Arch).first()
        pkg_stay.name = "pkg-test-stay"
        self.db.session.add(pkg_stay)
        self.db.session.flush()

        config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_stay.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_stay.has_lob("package"))

        # add build and package and lob that will be deleted
        build = Build()
        build.base_package_name = "build_unassigned"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)

        pkg_del = Package()
        pkg_del.build = build
        pkg_del.arch = self.db.session.query(Arch).first()
        pkg_del.name = "pkg-test-del"
        self.db.session.add(pkg_del)

        self.db.session.flush()

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_del.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_del.has_lob("package"))

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(self.call_action_ordered_args("cleanup-unassigned", [
            "--force"
        ]), 0)

        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra)

        self.assertFalse(pkg_del.has_lob("package"))
        self.assertTrue(pkg_stay.has_lob("package"))
Esempio n. 5
0
    def builds(self, count=10):
        self.begin("Builds")

        comps = self.ses.query(OpSysComponent).all()

        for comp in comps:
            for i in range(count):
                new = Build()
                new.component = comp
                new.epoch = 0
                new.version = "{0}.{1}".format(random.randrange(0, 5), random.randrange(0, 20))
                new.release = random.randrange(0, 100)
                self.add(new)

        self.commit()
Esempio n. 6
0
    def test_semver_stored_on_flush(self):
        """
        Make sure Build.semver is stored on session.flush
        """

        ver = "1.2.3-yeah666"
        build = Build()
        build.base_package_name = "sample"
        build.version = ver
        build.release = "1"
        build.epoch = "0"
        self.db.session.add(build)
        self.db.session.flush()

        self.assertEqual(build.version, build.semver)
        self.assertEqual(build.release, build.semrel)
Esempio n. 7
0
    def setUp(self):
        super(ProblemsTestCase, self).setUp()
        self.basic_fixtures()

        build = Build()
        build.base_package_name = "kernel"
        build.epoch = 0
        build.version = "3.12.10"
        build.release = "300.fc20"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_x86_64
        pkg.name = "kernel"
        self.db.session.add(pkg)

        self.db.session.commit()

        self.save_report("ureport_kerneloops")
        self.call_action("create-problems")

        self.db.session.commit()
Esempio n. 8
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if repo.opsys_list:  # parametrized
                self.log_info("Processing parametrized repo '{0}'"
                              .format(repo.name))

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture")
                    return 1
                try:
                    repo_instances += list(self._get_parametrized_variants(repo))
                except:
                    self.log_error("No valid mirror for repository {0}"
                                                    .format(repo.name))
                    return 1


            elif repo.opsysrelease_list:
                self.log_info("Processing repo '{0}' assigned with OpSysRelease"
                          .format(repo.name))

                if not repo.arch_list:
                    self.log_error("OpSysRelease repository is not assigned"
                                   " with an architecture")
                    return 1
                try:
                    repo_instances += list(self._get_opsysrelease_variants(repo))
                except:
                    self.log_error("No valid mirror for repository {0}"
                                                    .format(repo.name))
                    return 1

            else:
                if any('$' in url.url for url in repo.url_list):
                    self.log_error("No operating system assigned to"
                            "parametrized repo '{0}".format(repo.name))
                    return 1
                for arch in repo.arch_list:
                    try:
                        repo_instance = {
                                'instance' : repo_types[repo.type](repo.name,
                                            [url.url for url in repo.url_list]),
                                'opsys' : None,
                                'release' : None,
                                'arch' : arch.name}
                        repo_instances.append(repo_instance)
                    except:
                        self.log_error("No valid mirror for repository {0}"
                                                        .format(repo.name))
                        return 1

        cmdline.name_prefix = cmdline.name_prefix.lower()
        architectures = dict((x.name, x) for x in get_archs(db))
        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'"
                          .format(repo_instance['instance'].name,
                              repo_instance['instance'].urls))

            pkglist = \
                repo_instance['instance'].list_packages(architectures.keys())
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}"
                               .format(num + 1, total, pkg["name"]))

                if not pkg["name"].lower().startswith(cmdline.name_prefix):
                    self.log_debug("Skipped package {0}"
                                   .format(pkg["name"]))
                    continue
                arch = architectures.get(pkg["arch"], None)
                if not arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(pkg["arch"]))

                    continue

                repo_arch = architectures.get(repo_instance["arch"], None)
                if not repo_arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(repo_instance["arch"]))

                    continue

                build = (db.session.query(Build)
                         .filter(Build.base_package_name ==
                                 pkg["base_package_name"])
                         .filter(Build.version == pkg["version"])
                         .filter(Build.release == pkg["release"])
                         .filter(Build.epoch == pkg["epoch"])
                         .first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)
                    db.session.flush()

                build_arch = (db.session.query(BuildArch)
                         .filter(BuildArch.build_id == build.id)
                         .filter(BuildArch.arch_id == arch.id)
                         .first())

                if not build_arch:
                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                build_opsysrelease_arch = (
                       db.session.query(BuildOpSysReleaseArch)
                       .join(Build)
                       .join(OpSysRelease)
                       .join(Arch)
                       .filter(Build.id == build.id)
                       .filter(OpSys.name == repo_instance['opsys'])
                       .filter(OpSysRelease.version == repo_instance['release'])
                       .filter(Arch.name == repo_instance['arch'])
                       .first())

                if not build_opsysrelease_arch and repo_instance['release'] and repo_instance['opsys']:
                    self.log_info("Adding link between build {0}-{1} "
                            "operating system '{2}', release '{3} and "
                            "architecture {4}".format(pkg["base_package_name"],
                            pkg["version"], repo_instance['opsys'],
                            repo_instance['release'], repo_instance['arch']))

                    opsysrelease = (
                       db.session.query(OpSysRelease)
                       .filter(OpSys.name == repo_instance['opsys'])
                       .filter(OpSysRelease.version == repo_instance['release'])
                       .first())

                    bosra = BuildOpSysReleaseArch()
                    bosra.build = build
                    bosra.opsysrelease = opsysrelease
                    bosra.arch = repo_arch

                    db.session.add(bosra)
                    db.session.flush()

                package = (db.session.query(Package)
                           .filter(Package.name == pkg["name"])
                           .filter(Package.pkgtype == pkg["type"])
                           .filter(Package.build == build)
                           .filter(Package.arch == arch)
                           .first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attemps"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo.nogpgcheck)

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))
Esempio n. 9
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if "$" in repo.url:  # parametrized
                self.log_info("Processing parametrized repo '{0}'"
                              .format(repo.name))

                if not repo.opsys_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an operating system")
                    return 1

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture")
                    return 1

                repo_instances += list(self._get_parametrized_variants(repo))
            else:
                repo_instance = repo_types[repo.type](repo.name, repo.url)
                repo_instances.append(repo_instance)

        cmdline.name_prefix = cmdline.name_prefix.lower()
        architectures = dict((x.name, x) for x in get_archs(db))

        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'"
                          .format(repo_instance.name, repo_instance.urls[0]))

            pkglist = repo_instance.list_packages(architectures.keys())
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}"
                               .format(num + 1, total, pkg["name"]))

                if not pkg["name"].lower().startswith(cmdline.name_prefix):
                    self.log_debug("Skipped package {0}"
                                   .format(pkg["name"]))
                    continue

                arch = architectures.get(pkg["arch"], None)
                if not arch:
                    self.log_error("Architecture '{0}' not found, skipping"
                                   .format(pkg["arch"]))

                    continue

                build = (db.session.query(Build)
                         .filter(Build.base_package_name ==
                                 pkg["base_package_name"])
                         .filter(Build.version == pkg["version"])
                         .filter(Build.release == pkg["release"])
                         .filter(Build.epoch == pkg["epoch"])
                         .first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)

                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                package = (db.session.query(Package)
                           .filter(Package.name == pkg["name"])
                           .filter(Package.pkgtype == pkg["type"])
                           .filter(Package.build == build)
                           .filter(Package.arch == arch)
                           .first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attemps"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo.nogpgcheck)

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))
Esempio n. 10
0
    def releasedel_testing(self, repo_type):
        self.assertEqual(self.call_action("releasedel"), 1)
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "FooBarOS",
        }), 1)
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "fedora",
            "opsys-release": "1337",
        }), 1)

        # add release f24 and assign two builds
        self.assign_release_to_builds_testing(repo_type)

        # assign sample_repo to f24 release (sample_repo comes from ^^^)
        self.assertEqual(self.call_action_ordered_args("repoassign", [
            "sample_repo",  # NAME
            "Fedora 24",  # OPSYS
            "x86_64",  # ARCH
        ]), 0)

        tracker = Bugtracker(name="fedora-bugzilla")
        self.db.session.add(tracker)

        self.save_report('ureport_f20')
        self.save_report('ureport_core')
        self.call_action("create-problems")

        bzuser = BzUser(name="Fake user",
                        email="*****@*****.**",
                        real_name="Fake name",
                        can_login=False)

        self.db.session.add(bzuser)

        bug = BzBug()
        bug.id = 123456
        bug.summary = "Fake bug"
        bug.status = "NEW"
        bug.creation_time = datetime.now()
        bug.last_change_time = datetime.now()
        bug.whiteboard = "empty"
        bug.tracker = tracker
        bug.creator = bzuser
        bug.component = self.comp_faf
        bug.opsysrelease = self.release_20
        bug.private = False

        self.db.session.add(bug)

        # add package and lob that will be deleted
        pkg_del = Package()
        pkg_del.build = self.db.session.query(Build).first()
        pkg_del.arch = self.db.session.query(Arch).first()
        pkg_del.name = "pkg-test-del"
        self.db.session.add(pkg_del)
        self.db.session.flush()

        config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_del.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_del.has_lob("package"))

        # add build and package and lob that will not be deleted
        build = Build()
        build.base_package_name = "build_unassigned"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)

        pkg_stay = Package()
        pkg_stay.build = build
        pkg_stay.arch = self.db.session.query(Arch).first()
        pkg_stay.name = "pkg-test-stay"
        self.db.session.add(pkg_stay)
        self.db.session.flush()

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_stay.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_stay.has_lob("package"))

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()

        # delete release f24 with assigned repos and builds with lobs
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "fedora",
            "opsys-release": "24",
        }), 0)

        self.assertEqual(bosra, init_bosra)
        self.assertTrue(pkg_stay.has_lob("package"))
        self.assertFalse(pkg_del.has_lob("package"))

        self.assertEqual(self.db.session.query(Report).count(), 2)
        self.assertEqual(self.db.session.query(Problem).count(), 2)

        # delete release f20 with assigned bugs, reports and problems
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "fedora",
            "opsys-release": "20",
        }), 0)

        self.assertEqual(self.db.session.query(Report).count(), 0)
        self.assertEqual(self.db.session.query(Problem).count(), 0)

        # attempt to delete deleted release f20
        self.assertEqual(self.call_action("releasedel", {
            "opsys": "fedora",
            "opsys-release": "20",
        }), 1)
Esempio n. 11
0
    def test_get_packages_and_their_reports_unknown_packages(self):
        """
        """

        # add required stuff to db
        arch = Arch()
        arch.name = "noarch"
        self.db.session.add(arch)

        arch1 = Arch()
        arch1.name = "x86_64"
        self.db.session.add(arch1)

        build = Build()
        build.base_package_name = "sample"
        build.version = "1"
        build.release = "1"
        build.epoch = "0"
        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build
        self.db.session.add(pkg)
        #print(pkg.nevra())

        #different arch
        pkg2 = Package()
        pkg2.name = "sample"
        pkg2.pkgtype = "rpm"
        pkg2.arch = arch1
        pkg2.build = build
        self.db.session.add(pkg2)
        #print(pkg2.nevra())

        build2 = Build()
        build2.base_package_name = "sample"
        build2.version = "1"
        build2.release = "2"
        build2.epoch = "0"
        self.db.session.add(build2)

        pkg3 = Package()
        pkg3.name = "sample"
        pkg3.pkgtype = "rpm"
        pkg3.arch = arch
        pkg3.build = build2
        self.db.session.add(pkg3)
        #print(pkg3.nevra())

        problem = Problem()
        self.db.session.add(problem)

        opsys = OpSys()
        opsys.name = "Fedora"
        self.db.session.add(opsys)

        opsys_component = OpSysComponent()
        opsys_component.name = "core"
        opsys_component.opsys = opsys
        self.db.session.add(opsys_component)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = opsys_component
        self.db.session.add(report)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = pkg.name
        report_unknown.epoch = pkg.build.epoch
        report_unknown.version = pkg.build.version
        report_unknown.release = pkg.build.release
        report_unknown.arch = pkg.arch
        report_unknown.count = 1
        self.db.session.add(report_unknown)

        report_unknown2 = ReportUnknownPackage()
        report_unknown2.report = report
        report_unknown2.name = pkg2.name
        report_unknown2.epoch = pkg2.build.epoch
        report_unknown2.version = pkg2.build.version
        report_unknown2.release = pkg2.build.release
        report_unknown2.arch = pkg2.arch
        report_unknown2.count = 1
        self.db.session.add(report_unknown2)

        report_unknown3 = ReportUnknownPackage()
        report_unknown3.report = report
        report_unknown3.name = "nonsense"
        report_unknown3.epoch = pkg.build.epoch
        report_unknown3.version = pkg.build.version
        report_unknown3.release = pkg.build.release
        report_unknown3.arch = pkg.arch
        report_unknown3.count = 1
        self.db.session.add(report_unknown3)

        self.db.session.flush()

        packages_and_their_reports_unknown_packages = \
            get_packages_and_their_reports_unknown_packages(self.db).all()
        self.assertEqual(len(packages_and_their_reports_unknown_packages), 2)
        self.assertIn(
            (pkg, report_unknown), packages_and_their_reports_unknown_packages)
        self.assertIn(
            (pkg2, report_unknown2), packages_and_their_reports_unknown_packages)
Esempio n. 12
0
    def assign_release_to_builds_testing(self, repo_type):
        # add repo
        self.assertEqual(
            self.call_action_ordered_args(
                "repoadd",
                [
                    "sample_repo",  # NAME
                    repo_type,  # TYPE
                    "file:///sample_rpms",  # URL
                ]),
            0)

        # add release
        self.assertEqual(
            self.call_action("releaseadd", {
                "opsys": "fedora",
                "opsys-release": "24",
                "status": "ACTIVE",
            }), 0)

        # add two builds
        build = Build()
        build.base_package_name = "build"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc24"
        self.db.session.add(build)
        self.db.session.flush()

        build_arch = BuildArch(arch_id=self.arch_x86_64.id, build_id=build.id)
        self.db.session.add(build_arch)
        self.db.session.flush()

        build = Build()
        build.base_package_name = "build1"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)
        self.db.session.flush()

        build_arch = BuildArch(arch_id=self.arch_x86_64.id, build_id=build.id)

        self.db.session.add(build_arch)
        self.db.session.flush()

        systems['fedora'].get_released_builds = get_released_builds_mock

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(
            self.call_action_ordered_args(
                "assign-release-to-builds",
                [
                    "Fedora",  # OPSYS
                    "24",  # RELEASE
                    "x86_64",  # ARCH
                    "--expression=fc24",  # variant
                ]),
            0)
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra + 1)

        self.assertEqual(
            self.call_action_ordered_args(
                "assign-release-to-builds",
                [
                    "Fedora",  # OPSYS
                    "24",  # RELEASE
                    "x86_64",  # ARCH
                    "--released-builds",  # variant
                ]),
            0)
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        self.assertEqual(bosra, init_bosra + 2)
Esempio n. 13
0
    def releasedel_testing(self, repo_type):
        self.assertEqual(
            self.call_action("releasedel", {
                "opsys": "fedora",
                "opsys-release": "1337",
            }), 1)

        # add release f24 and assign two builds
        self.assign_release_to_builds_testing(repo_type)

        # assign sample_repo to f24 release (sample_repo comes from ^^^)
        self.assertEqual(
            self.call_action_ordered_args(
                "repoassign",
                [
                    "sample_repo",  # NAME
                    "Fedora 24",  # OPSYS
                    "x86_64",  # ARCH
                ]),
            0)

        tracker = Bugtracker(name="fedora-bugzilla")
        self.db.session.add(tracker)

        self.save_report('ureport_f20')
        self.save_report('ureport_core')
        self.call_action("create-problems")

        bzuser = BzUser(name="Fake user",
                        email="*****@*****.**",
                        real_name="Fake name",
                        can_login=False)

        self.db.session.add(bzuser)

        bug = BzBug()
        bug.id = 123456
        bug.summary = "Fake bug"
        bug.status = "NEW"
        bug.creation_time = datetime.now()
        bug.last_change_time = datetime.now()
        bug.whiteboard = "empty"
        bug.tracker = tracker
        bug.creator = bzuser
        bug.component = self.comp_faf
        bug.opsysrelease = self.release_20
        bug.private = False

        self.db.session.add(bug)

        # add package and lob that will be deleted
        pkg_del = Package()
        pkg_del.build = self.db.session.query(Build).first()
        pkg_del.arch = self.db.session.query(Arch).first()
        pkg_del.name = "pkg-test-del"
        self.db.session.add(pkg_del)
        self.db.session.flush()

        config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_del.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_del.has_lob("package"))

        # add build and package and lob that will not be deleted
        build = Build()
        build.base_package_name = "build_unassigned"
        build.epoch = 0
        build.version = "1.2.3"
        build.release = "20.fc23"
        self.db.session.add(build)

        pkg_stay = Package()
        pkg_stay.build = build
        pkg_stay.arch = self.db.session.query(Arch).first()
        pkg_stay.name = "pkg-test-stay"
        self.db.session.add(pkg_stay)
        self.db.session.flush()

        sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
        with open(sample_rpm, mode='rb') as sample:
            pkg_stay.save_lob("package", sample, truncate=True)
        self.assertTrue(pkg_stay.has_lob("package"))

        init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
        bosra = self.db.session.query(BuildOpSysReleaseArch).count()

        # delete release f24 with assigned repos and builds with lobs
        self.assertEqual(
            self.call_action("releasedel", {
                "opsys": "fedora",
                "opsys-release": "24",
            }), 0)

        self.assertEqual(bosra, init_bosra)
        self.assertTrue(pkg_stay.has_lob("package"))
        self.assertFalse(pkg_del.has_lob("package"))

        self.assertEqual(self.db.session.query(Report).count(), 2)
        self.assertEqual(self.db.session.query(Problem).count(), 2)

        # delete release f20 with assigned bugs, reports and problems
        self.assertEqual(
            self.call_action("releasedel", {
                "opsys": "fedora",
                "opsys-release": "20",
            }), 0)

        self.assertEqual(self.db.session.query(Report).count(), 0)
        self.assertEqual(self.db.session.query(Problem).count(), 0)

        # attempt to delete deleted release f20
        self.assertEqual(
            self.call_action("releasedel", {
                "opsys": "fedora",
                "opsys-release": "20",
            }), 1)
Esempio n. 14
0
    def test_get_packages_and_their_reports_unknown_packages(self):
        """
        """

        # add required stuff to db
        arch = Arch()
        arch.name = "noarch"
        self.db.session.add(arch)

        arch1 = Arch()
        arch1.name = "x86_64"
        self.db.session.add(arch1)

        build = Build()
        build.base_package_name = "sample"
        build.version = "1"
        build.release = "1"
        build.semver = "1.0.0"
        build.semrel = "1.0.0"
        build.epoch = "0"
        self.db.session.add(build)

        pkg = Package()
        pkg.name = "sample"
        pkg.pkgtype = "rpm"
        pkg.arch = arch
        pkg.build = build
        self.db.session.add(pkg)
        #print(pkg.nevra())

        #different arch
        pkg2 = Package()
        pkg2.name = "sample"
        pkg2.pkgtype = "rpm"
        pkg2.arch = arch1
        pkg2.build = build
        self.db.session.add(pkg2)
        #print(pkg2.nevra())

        build2 = Build()
        build2.base_package_name = "sample"
        build2.version = "1"
        build2.release = "2"
        build2.epoch = "0"
        self.db.session.add(build2)

        pkg3 = Package()
        pkg3.name = "sample"
        pkg3.pkgtype = "rpm"
        pkg3.arch = arch
        pkg3.build = build2
        self.db.session.add(pkg3)
        #print(pkg3.nevra())

        problem = Problem()
        self.db.session.add(problem)

        opsys = OpSys()
        opsys.name = "Fedora"
        self.db.session.add(opsys)

        opsys_component = OpSysComponent()
        opsys_component.name = "core"
        opsys_component.opsys = opsys
        self.db.session.add(opsys_component)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = opsys_component
        self.db.session.add(report)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = pkg.name
        report_unknown.epoch = pkg.build.epoch
        report_unknown.version = pkg.build.version
        report_unknown.release = pkg.build.release
        report_unknown.semver = pkg.build.semver
        report_unknown.semrel = pkg.build.semrel
        report_unknown.arch = pkg.arch
        report_unknown.count = 1
        self.db.session.add(report_unknown)

        report_unknown2 = ReportUnknownPackage()
        report_unknown2.report = report
        report_unknown2.name = pkg2.name
        report_unknown2.epoch = pkg2.build.epoch
        report_unknown2.version = pkg2.build.version
        report_unknown2.release = pkg2.build.release
        report_unknown2.semver = pkg2.build.semver
        report_unknown2.semrel = pkg2.build.semrel
        report_unknown2.arch = pkg2.arch
        report_unknown2.count = 1
        self.db.session.add(report_unknown2)

        report_unknown3 = ReportUnknownPackage()
        report_unknown3.report = report
        report_unknown3.name = "nonsense"
        report_unknown3.epoch = pkg.build.epoch
        report_unknown3.version = pkg.build.version
        report_unknown3.release = pkg.build.release
        report_unknown3.semver = pkg.build.semver
        report_unknown3.semrel = pkg.build.semrel
        report_unknown3.arch = pkg.arch
        report_unknown3.count = 1
        self.db.session.add(report_unknown3)

        self.db.session.flush()

        packages_and_their_reports_unknown_packages = \
            get_packages_and_their_reports_unknown_packages(self.db).all()
        self.assertEqual(len(packages_and_their_reports_unknown_packages), 2)
        self.assertIn((pkg, report_unknown),
                      packages_and_their_reports_unknown_packages)
        self.assertIn((pkg2, report_unknown2),
                      packages_and_their_reports_unknown_packages)
Esempio n. 15
0
    def run(self, cmdline, db):
        repo_instances = []

        for repo in db.session.query(Repo):
            if cmdline.NAME and repo.name not in cmdline.NAME:
                continue

            if not repo.type in repo_types:
                self.log_error("No plugin installed to handle repository type "
                               "{0}, skipping.".format(repo.type))
                continue

            if repo.opsys_list:  # parametrized
                self.log_info("Processing parametrized repo '{0}'".format(
                    repo.name))

                if not repo.arch_list:
                    self.log_error("Parametrized repository is not assigned"
                                   " with an architecture, skipping")
                    continue
                try:
                    repo_instances += list(
                        self._get_parametrized_variants(repo))
                except:
                    self.log_error(
                        "No valid mirror for repository '{0}', skipping".
                        format(repo.name))
                    continue

            elif repo.opsysrelease_list:
                self.log_info(
                    "Processing repo '{0}' assigned with OpSysRelease".format(
                        repo.name))

                if not repo.arch_list:
                    self.log_error("OpSysRelease repository is not assigned"
                                   " with an architecture, skipping")
                    continue
                try:
                    repo_instances += list(
                        self._get_opsysrelease_variants(repo))
                except:
                    self.log_error(
                        "No valid mirror for repository '{0}', skipping".
                        format(repo.name))
                    continue

            else:
                if any('$' in url.url for url in repo.url_list):
                    self.log_error("No operating system assigned to"
                                   "parametrized repo '{0}', skipping".format(
                                       repo.name))
                    continue
                for arch in repo.arch_list:
                    try:
                        repo_instance = {
                            'instance':
                            repo_types[repo.type](
                                repo.name, [url.url for url in repo.url_list]),
                            'opsys':
                            None,
                            'release':
                            None,
                            'arch':
                            arch.name
                        }
                        repo_instances.append(repo_instance)
                    except:
                        self.log_error(
                            "No valid mirror for repository '{0}', skipping".
                            format(repo.name))
                        continue

        cmdline.name_prefix = cmdline.name_prefix.lower()
        architectures = dict((x.name, x) for x in get_archs(db))
        for repo_instance in repo_instances:
            self.log_info("Processing repository '{0}' URL: '{1}'".format(
                repo_instance['instance'].name,
                repo_instance['instance'].urls))

            pkglist = \
                repo_instance['instance'].list_packages(architectures.keys())
            total = len(pkglist)

            self.log_info("Repository has '{0}' packages".format(total))

            for num, pkg in enumerate(pkglist):
                self.log_debug("[{0} / {1}] Processing package {2}".format(
                    num + 1, total, pkg["name"]))

                if not pkg["name"].lower().startswith(cmdline.name_prefix):
                    self.log_debug("Skipped package {0}".format(pkg["name"]))
                    continue
                arch = architectures.get(pkg["arch"], None)
                if not arch:
                    self.log_error(
                        "Architecture '{0}' not found, skipping".format(
                            pkg["arch"]))

                    continue

                repo_arch = architectures.get(repo_instance["arch"], None)
                if not repo_arch:
                    self.log_error(
                        "Architecture '{0}' not found, skipping".format(
                            repo_instance["arch"]))

                    continue

                build = (db.session.query(Build).filter(
                    Build.base_package_name == pkg["base_package_name"]).
                         filter(Build.version == pkg["version"]).filter(
                             Build.release == pkg["release"]).filter(
                                 Build.epoch == pkg["epoch"]).first())

                if not build:
                    self.log_debug("Adding build {0}-{1}".format(
                        pkg["base_package_name"], pkg["version"]))

                    build = Build()
                    build.base_package_name = pkg["base_package_name"]
                    build.version = pkg["version"]
                    build.release = pkg["release"]
                    build.epoch = pkg["epoch"]

                    db.session.add(build)
                    db.session.flush()

                build_arch = (db.session.query(BuildArch).filter(
                    BuildArch.build_id == build.id).filter(
                        BuildArch.arch_id == arch.id).first())

                if not build_arch:
                    build_arch = BuildArch()
                    build_arch.build = build
                    build_arch.arch = arch

                    db.session.add(build_arch)
                    db.session.flush()

                build_opsysrelease_arch = (db.session.query(
                    BuildOpSysReleaseArch
                ).join(Build).join(OpSysRelease).join(Arch).filter(
                    Build.id == build.id).filter(
                        OpSys.name == repo_instance['opsys']).filter(
                            OpSysRelease.version == repo_instance['release']
                        ).filter(Arch.name == repo_instance['arch']).first())

                if not build_opsysrelease_arch and repo_instance[
                        'release'] and repo_instance['opsys']:
                    self.log_info("Adding link between build {0}-{1} "
                                  "operating system '{2}', release '{3} and "
                                  "architecture {4}".format(
                                      pkg["base_package_name"], pkg["version"],
                                      repo_instance['opsys'],
                                      repo_instance['release'],
                                      repo_instance['arch']))

                    opsysrelease = (db.session.query(OpSysRelease).filter(
                        OpSys.name == repo_instance['opsys']).filter(
                            OpSysRelease.version ==
                            repo_instance['release']).first())

                    bosra = BuildOpSysReleaseArch()
                    bosra.build = build
                    bosra.opsysrelease = opsysrelease
                    bosra.arch = repo_arch

                    db.session.add(bosra)
                    db.session.flush()

                package = (db.session.query(Package).filter(
                    Package.name == pkg["name"]).filter(
                        Package.pkgtype == pkg["type"]).filter(
                            Package.build == build).filter(
                                Package.arch == arch).first())

                if not package:
                    self.log_info("Adding package {0}".format(pkg["filename"]))

                    package = Package()
                    package.name = pkg["name"]
                    package.pkgtype = pkg["type"]
                    package.arch = arch
                    package.build = build

                    db.session.add(package)
                    db.session.flush()

                    if cmdline.no_download_rpm:
                        continue

                    # Catching too general exception Exception
                    # pylint: disable-msg=W0703
                    try:
                        self.log_info("Downloading {0}".format(pkg["url"]))
                        self._download(package, "package", pkg["url"])
                    except Exception as exc:
                        self.log_error("Exception ({0}) after multiple attemps"
                                       " while trying to download {1},"
                                       " skipping.".format(exc, pkg["url"]))

                        db.session.delete(package)
                        db.session.flush()
                        continue
                    # pylint: enable-msg=W0703

                    res = True
                    if pkg["type"] == "rpm":
                        res = store_rpm_deps(db, package, repo.nogpgcheck)

                    if not res:
                        self.log_error("Post-processing failed, skipping")
                        db.session.delete(package)
                        db.session.flush()
                        continue

                    if cmdline.no_store_rpm:
                        try:
                            package.del_lob("package")
                            self.log_info("Package deleted.")
                        except Exception as exc:
                            self.log_error("Error deleting the RPM file.")

                else:
                    self.log_debug("Known package {0}".format(pkg["filename"]))
Esempio n. 16
0
    def setUp(self):
        super(MPFTestCase, self).setUp()
        self.basic_fixtures()

        # Fixed problem 1
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_systemd
        self.db.session.add(report)

        with open("sample_reports/ureport_systemd2", "r") as file:
            self.ureport_systemd2 = json.load(file)
        with open("sample_reports/ureport_systemd77", "r") as file:
            self.ureport_systemd77 = json.load(file)
        problemplugin = problemtypes[self.ureport_systemd2["problem"]["type"]]
        report_hash = problemplugin.hash_ureport(
            self.ureport_systemd2["problem"])
        db_hash = ReportHash()
        db_hash.hash = report_hash
        db_hash.report = report
        self.db.session.add(db_hash)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = "systemd-test"
        report_unknown.epoch = 0
        report_unknown.version = 2
        report_unknown.release = 1
        report_unknown.semver = "2.0.0"
        report_unknown.semrel = "1.0.0"
        report_unknown.arch = self.arch_i686
        report_unknown.count = 1
        report_unknown.type = "CRASHED"
        self.db.session.add(report_unknown)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = "faf"
        report_unknown.epoch = 0
        report_unknown.version = 1
        report_unknown.release = 1
        report_unknown.semver = "1.0.0"
        report_unknown.semrel = "1.0.0"
        report_unknown.arch = self.arch_i686
        report_unknown.count = 1
        report_unknown.type = "RELATED"
        self.db.session.add(report_unknown)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_19
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_fixed1 = problem

        # Fixed problem 2
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_faf
        self.db.session.add(report)

        build = Build()
        build.base_package_name = "faf"
        build.epoch = 0
        build.version = "3.4"
        build.release = "5"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_i686
        pkg.name = "faf-test"
        self.db.session.add(pkg)

        rpkg = ReportPackage()
        rpkg.report = report
        rpkg.type = "CRASHED"
        rpkg.installed_package = pkg
        rpkg.count = 66
        self.db.session.add(rpkg)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_fixed2 = problem

        # Unfixed problem 1
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_systemd
        self.db.session.add(report)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = "systemd"
        report_unknown.epoch = 0
        report_unknown.version = 3
        report_unknown.release = 1
        report_unknown.semver = "3.0.0"
        report_unknown.semrel = "1.0.0"
        report_unknown.arch = self.arch_i686
        report_unknown.count = 1
        report_unknown.type = "CRASHED"
        self.db.session.add(report_unknown)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_unfixed1 = problem

        # Unfixed problem 2
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_systemd
        self.db.session.add(report)

        report_unknown = ReportUnknownPackage()
        report_unknown.report = report
        report_unknown.name = "systemd"
        report_unknown.epoch = 0
        report_unknown.version = 3
        report_unknown.release = 0
        report_unknown.semver = "3.0.0"
        report_unknown.semrel = "0.0.0"
        report_unknown.arch = self.arch_i686
        report_unknown.count = 1
        report_unknown.type = "CRASHED"
        self.db.session.add(report_unknown)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_unfixed2 = problem

        # Unfixed problem 3
        problem = Problem()
        self.db.session.add(problem)

        report = Report()
        report.type = "type"
        report.count = 2
        report.problem = problem
        report.component = self.comp_faf
        self.db.session.add(report)

        build = Build()
        build.base_package_name = "faf"
        build.epoch = 0
        build.version = "3.110"
        build.release = "5"
        self.db.session.add(build)

        pkg = Package()
        pkg.build = build
        pkg.arch = self.arch_i686
        pkg.name = "faf-test"
        self.db.session.add(pkg)

        rpkg = ReportPackage()
        rpkg.report = report
        rpkg.type = "CRASHED"
        rpkg.installed_package = pkg
        rpkg.count = 66
        self.db.session.add(rpkg)

        report_osr = ReportOpSysRelease()
        report_osr.count = 77
        report_osr.opsysrelease = self.release_20
        report_osr.report = report
        self.db.session.add(report_osr)

        self.problem_unfixed3 = problem

        self.db.session.flush()

        systems['fedora'].get_released_builds = get_released_builds_mock
        systems['fedora'].build_aging_days = 7