def cleanup_packages_testing_arch(self, repo_type): self.assign_release_to_builds_testing(repo_type) # add package and lob pkg = Package() pkg.build = self.db.session.query(Build).first() pkg.arch = self.db.session.query(Arch).first() pkg.name = "pkg-test" self.db.session.add(pkg) self.db.session.flush() config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf") sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg.save_lob("package", sample, truncate=True) self.assertTrue(pkg.has_lob("package")) init_bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual( self.call_action("cleanup-packages", { "arch": "noarch", }), 0) bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual(bosra, init_bosra) self.assertEqual( self.call_action("cleanup-packages", { "arch": "x86_64", }), 0) bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual(bosra, init_bosra - 2) self.assertFalse(pkg.has_lob("package"))
def cleanup_packages_testing(self, repo_type): self.assign_release_to_builds_testing(repo_type) # add package and lob pkg = Package() pkg.build = self.db.session.query(Build).first() pkg.arch = self.db.session.query(Arch).first() pkg.name = "pkg-test" self.db.session.add(pkg) self.db.session.flush() config["storage.lobdir"] = "/tmp/faf_test_data/lob" sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg.save_lob("package", sample, truncate=True) self.assertTrue(pkg.has_lob("package")) init_bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual(self.call_action_ordered_args("cleanup-packages", [ "Fedora", # OPSYS "24", # RELEASE ]), 0) bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual(bosra, init_bosra - 2) self.assertFalse(pkg.has_lob("package"))
def cleanup_packages_testing(self, repo_type): self.assign_release_to_builds_testing(repo_type) # add package and lob pkg = Package() pkg.build = self.db.session.query(Build).first() pkg.arch = self.db.session.query(Arch).first() pkg.name = "pkg-test" self.db.session.add(pkg) self.db.session.flush() config["storage.lobdir"] = "/tmp/faf_test_data/lob" sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg.save_lob("package", sample, truncate=True) self.assertTrue(pkg.has_lob("package")) init_bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual( self.call_action_ordered_args( "cleanup-packages", [ "Fedora", # OPSYS "24", # RELEASE ]), 0) bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual(bosra, init_bosra - 2) self.assertFalse(pkg.has_lob("package"))
def cleanup_unassigned_testing(self, repo_type): self.assign_release_to_builds_testing(repo_type) # add package and lob that will not be deleted pkg_stay = Package() pkg_stay.build = self.db.session.query(Build).first() pkg_stay.arch = self.db.session.query(Arch).first() pkg_stay.name = "pkg-test-stay" self.db.session.add(pkg_stay) self.db.session.flush() config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf") sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg_stay.save_lob("package", sample, truncate=True) self.assertTrue(pkg_stay.has_lob("package")) # add build and package and lob that will be deleted build = Build() build.base_package_name = "build_unassigned" build.epoch = 0 build.version = "1.2.3" build.release = "20.fc23" self.db.session.add(build) pkg_del = Package() pkg_del.build = build pkg_del.arch = self.db.session.query(Arch).first() pkg_del.name = "pkg-test-del" self.db.session.add(pkg_del) self.db.session.flush() sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg_del.save_lob("package", sample, truncate=True) self.assertTrue(pkg_del.has_lob("package")) init_bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual( self.call_action_ordered_args("cleanup-unassigned", ["--force"]), 0) bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual(bosra, init_bosra) self.assertFalse(pkg_del.has_lob("package")) self.assertTrue(pkg_stay.has_lob("package"))
def cleanup_unassigned_testing(self, repo_type): self.assign_release_to_builds_testing(repo_type) # add package and lob that will not be deleted pkg_stay = Package() pkg_stay.build = self.db.session.query(Build).first() pkg_stay.arch = self.db.session.query(Arch).first() pkg_stay.name = "pkg-test-stay" self.db.session.add(pkg_stay) self.db.session.flush() config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf") sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg_stay.save_lob("package", sample, truncate=True) self.assertTrue(pkg_stay.has_lob("package")) # add build and package and lob that will be deleted build = Build() build.base_package_name = "build_unassigned" build.epoch = 0 build.version = "1.2.3" build.release = "20.fc23" self.db.session.add(build) pkg_del = Package() pkg_del.build = build pkg_del.arch = self.db.session.query(Arch).first() pkg_del.name = "pkg-test-del" self.db.session.add(pkg_del) self.db.session.flush() sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg_del.save_lob("package", sample, truncate=True) self.assertTrue(pkg_del.has_lob("package")) init_bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual(self.call_action_ordered_args("cleanup-unassigned", [ "--force" ]), 0) bosra = self.db.session.query(BuildOpSysReleaseArch).count() self.assertEqual(bosra, init_bosra) self.assertFalse(pkg_del.has_lob("package")) self.assertTrue(pkg_stay.has_lob("package"))
def releasedel_testing(self, repo_type): self.assertEqual( self.call_action("releasedel", { "opsys": "fedora", "opsys-release": "1337", }), 1) # add release f24 and assign two builds self.assign_release_to_builds_testing(repo_type) # assign sample_repo to f24 release (sample_repo comes from ^^^) self.assertEqual( self.call_action_ordered_args( "repoassign", [ "sample_repo", # NAME "Fedora 24", # OPSYS "x86_64", # ARCH ]), 0) tracker = Bugtracker(name="fedora-bugzilla") self.db.session.add(tracker) self.save_report('ureport_f20') self.save_report('ureport_core') self.call_action("create-problems") bzuser = BzUser(name="Fake user", email="*****@*****.**", real_name="Fake name", can_login=False) self.db.session.add(bzuser) bug = BzBug() bug.id = 123456 bug.summary = "Fake bug" bug.status = "NEW" bug.creation_time = datetime.now() bug.last_change_time = datetime.now() bug.whiteboard = "empty" bug.tracker = tracker bug.creator = bzuser bug.component = self.comp_faf bug.opsysrelease = self.release_20 bug.private = False self.db.session.add(bug) # add package and lob that will be deleted pkg_del = Package() pkg_del.build = self.db.session.query(Build).first() pkg_del.arch = self.db.session.query(Arch).first() pkg_del.name = "pkg-test-del" self.db.session.add(pkg_del) self.db.session.flush() config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf") sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg_del.save_lob("package", sample, truncate=True) self.assertTrue(pkg_del.has_lob("package")) # add build and package and lob that will not be deleted build = Build() build.base_package_name = "build_unassigned" build.epoch = 0 build.version = "1.2.3" build.release = "20.fc23" self.db.session.add(build) pkg_stay = Package() pkg_stay.build = build pkg_stay.arch = self.db.session.query(Arch).first() pkg_stay.name = "pkg-test-stay" self.db.session.add(pkg_stay) self.db.session.flush() sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg_stay.save_lob("package", sample, truncate=True) self.assertTrue(pkg_stay.has_lob("package")) init_bosra = self.db.session.query(BuildOpSysReleaseArch).count() bosra = self.db.session.query(BuildOpSysReleaseArch).count() # delete release f24 with assigned repos and builds with lobs self.assertEqual( self.call_action("releasedel", { "opsys": "fedora", "opsys-release": "24", }), 0) self.assertEqual(bosra, init_bosra) self.assertTrue(pkg_stay.has_lob("package")) self.assertFalse(pkg_del.has_lob("package")) self.assertEqual(self.db.session.query(Report).count(), 2) self.assertEqual(self.db.session.query(Problem).count(), 2) # delete release f20 with assigned bugs, reports and problems self.assertEqual( self.call_action("releasedel", { "opsys": "fedora", "opsys-release": "20", }), 0) self.assertEqual(self.db.session.query(Report).count(), 0) self.assertEqual(self.db.session.query(Problem).count(), 0) # attempt to delete deleted release f20 self.assertEqual( self.call_action("releasedel", { "opsys": "fedora", "opsys-release": "20", }), 1)
def run(self, cmdline, db) -> None: repo_instances = [] for repo in db.session.query(Repo): if cmdline.REPO and repo.name not in cmdline.REPO: continue if cmdline.match_repos not in repo.name: continue if not repo.type in repo_types: self.log_error("No plugin installed to handle repository type " "{0}, skipping.".format(repo.type)) continue if repo.opsys_list: # parametrized self.log_info("Processing parametrized repo '{0}'".format( repo.name)) if not repo.arch_list: self.log_error("Parametrized repository is not assigned" " with an architecture, skipping") continue try: repo_instances += list( self._get_parametrized_variants(repo)) except: # pylint: disable=bare-except self.log_error( "No valid mirror for repository '{0}', skipping". format(repo.name)) continue elif repo.opsysrelease_list: self.log_info( "Processing repo '{0}' assigned with OpSysRelease".format( repo.name)) if not repo.arch_list: self.log_error("OpSysRelease repository is not assigned" " with an architecture, skipping") continue try: repo_instances += list( self._get_opsysrelease_variants(repo)) except: # pylint: disable=bare-except self.log_error( "No valid mirror for repository '{0}', skipping". format(repo.name)) continue else: if any('$' in url.url for url in repo.url_list): self.log_error("No operating system assigned to" "parametrized repo '{0}', skipping".format( repo.name)) continue for arch in repo.arch_list: try: repo_instance = { 'instance': repo_types[repo.type]( repo.name, [url.url for url in repo.url_list]), 'opsys': None, 'release': None, 'arch': arch.name, 'nogpgcheck': repo.nogpgcheck } repo_instances.append(repo_instance) except: # pylint: disable=bare-except self.log_error( "No valid mirror for repository '{0}', skipping". format(repo.name)) continue cmdline.name_prefix = cmdline.name_prefix.lower() architectures = dict((x.name, x) for x in get_archs(db)) for repo_instance in repo_instances: self.log_info( "Processing repository '{0}' assigned to OS release '{1}' and arch '{2}', URL: '{3}'" .format(repo_instance['instance'].name, repo_instance['release'], repo_instance['arch'], repo_instance['instance'].urls)) if cmdline.no_cache: repo_instance['instance'].cache_lifetime = 0 pkglist = \ repo_instance['instance'].list_packages(list(architectures.keys())) total = len(pkglist) self.log_info("Repository has {0} packages".format(total)) for num, pkg in enumerate(pkglist, start=1): self.log_info("[%d / %d] Processing package %s", num, total, pkg["name"]) if not pkg["name"].lower().startswith(cmdline.name_prefix): self.log_debug("Skipped package %s", pkg["name"]) continue arch = architectures.get(pkg["arch"], None) if not arch: self.log_error( "Architecture '{0}' not found, skipping".format( pkg["arch"])) continue repo_arch = architectures.get(repo_instance["arch"], None) if not repo_arch: self.log_error( "Architecture '{0}' not found, skipping".format( repo_instance["arch"])) continue build = (db.session.query(Build).filter( Build.base_package_name == pkg["base_package_name"]). filter(Build.version == pkg["version"]).filter( Build.release == pkg["release"]).filter( Build.epoch == pkg["epoch"]).first()) if not build: self.log_debug("Adding build %s-%s", pkg["base_package_name"], pkg["version"]) build = Build() build.base_package_name = pkg["base_package_name"] build.version = pkg["version"] build.release = pkg["release"] build.epoch = pkg["epoch"] db.session.add(build) db.session.flush() build_arch = (db.session.query(BuildArch).filter( BuildArch.build_id == build.id).filter( BuildArch.arch_id == arch.id).first()) if not build_arch: build_arch = BuildArch() build_arch.build = build build_arch.arch = arch db.session.add(build_arch) db.session.flush() build_opsysrelease_arch = (db.session.query( BuildOpSysReleaseArch ).join(Build).join(OpSysRelease).join(Arch).filter( Build.id == build.id).filter( OpSys.name == repo_instance['opsys']).filter( OpSysRelease.version == repo_instance['release'] ).filter(Arch.name == repo_instance['arch']).first()) if not build_opsysrelease_arch and repo_instance[ 'release'] and repo_instance['opsys']: self.log_info("Adding link between build '{0}-{1}' " "and operating system '{2} {3} {4}'".format( pkg["base_package_name"], pkg["version"], repo_instance['opsys'], repo_instance['release'], repo_instance['arch'])) opsysrelease = (db.session.query(OpSysRelease).filter( OpSys.name == repo_instance['opsys']).filter( OpSysRelease.version == repo_instance['release']).first()) bosra = BuildOpSysReleaseArch() bosra.build = build bosra.opsysrelease = opsysrelease bosra.arch = repo_arch db.session.add(bosra) db.session.flush() package = (db.session.query(Package).filter( Package.name == pkg["name"]).filter( Package.pkgtype == pkg["type"]).filter( Package.build == build).filter( Package.arch == arch).first()) if not package: self.log_info("Adding package {0}".format(pkg["filename"])) package = Package() package.name = pkg["name"] package.pkgtype = pkg["type"] package.arch = arch package.build = build db.session.add(package) db.session.flush() if cmdline.no_download_rpm: continue # Catching too general exception Exception # pylint: disable-msg=W0703 try: self.log_info("Downloading {0}".format(pkg["url"])) self._download(package, "package", pkg["url"]) except Exception as exc: self.log_error( "Exception ({0}) after multiple attempts" " while trying to download {1}," " skipping.".format(exc, pkg["url"])) db.session.delete(package) db.session.flush() continue # pylint: enable-msg=W0703 if pkg["type"] == "rpm": try: store_rpm_provides(db, package, repo_instance['nogpgcheck']) except FafError as ex: self.log_error( "Post-processing failed, skipping: {}".format( ex)) db.session.delete(package) db.session.flush() continue if cmdline.no_store_rpm: try: package.del_lob("package") self.log_info("Package deleted.") except Exception as exc: # pylint: disable=broad-except self.log_error("Error deleting the RPM file.") else: self.log_debug("Known package %s", pkg["filename"]) if not package.has_lob( "package") and not cmdline.no_download_rpm: self.log_info( "Package {} does not have a LOB. Re-downloading.". format(pkg["name"])) try: self._download(package, "package", pkg["url"]) except (FafError, URLError) as exc: self.log_error( "Exception ({0}) after multiple attempts" " while trying to download {1}," " skipping.".format(exc, pkg["url"]))
def releasedel_testing(self, repo_type): self.assertEqual(self.call_action("releasedel"), 1) self.assertEqual(self.call_action("releasedel", { "opsys": "FooBarOS", }), 1) self.assertEqual(self.call_action("releasedel", { "opsys": "fedora", "opsys-release": "1337", }), 1) # add release f24 and assign two builds self.assign_release_to_builds_testing(repo_type) # assign sample_repo to f24 release (sample_repo comes from ^^^) self.assertEqual(self.call_action_ordered_args("repoassign", [ "sample_repo", # NAME "Fedora 24", # OPSYS "x86_64", # ARCH ]), 0) tracker = Bugtracker(name="fedora-bugzilla") self.db.session.add(tracker) self.save_report('ureport_f20') self.save_report('ureport_core') self.call_action("create-problems") bzuser = BzUser(name="Fake user", email="*****@*****.**", real_name="Fake name", can_login=False) self.db.session.add(bzuser) bug = BzBug() bug.id = 123456 bug.summary = "Fake bug" bug.status = "NEW" bug.creation_time = datetime.now() bug.last_change_time = datetime.now() bug.whiteboard = "empty" bug.tracker = tracker bug.creator = bzuser bug.component = self.comp_faf bug.opsysrelease = self.release_20 bug.private = False self.db.session.add(bug) # add package and lob that will be deleted pkg_del = Package() pkg_del.build = self.db.session.query(Build).first() pkg_del.arch = self.db.session.query(Arch).first() pkg_del.name = "pkg-test-del" self.db.session.add(pkg_del) self.db.session.flush() config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf") sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg_del.save_lob("package", sample, truncate=True) self.assertTrue(pkg_del.has_lob("package")) # add build and package and lob that will not be deleted build = Build() build.base_package_name = "build_unassigned" build.epoch = 0 build.version = "1.2.3" build.release = "20.fc23" self.db.session.add(build) pkg_stay = Package() pkg_stay.build = build pkg_stay.arch = self.db.session.query(Arch).first() pkg_stay.name = "pkg-test-stay" self.db.session.add(pkg_stay) self.db.session.flush() sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0] with open(sample_rpm, mode='rb') as sample: pkg_stay.save_lob("package", sample, truncate=True) self.assertTrue(pkg_stay.has_lob("package")) init_bosra = self.db.session.query(BuildOpSysReleaseArch).count() bosra = self.db.session.query(BuildOpSysReleaseArch).count() # delete release f24 with assigned repos and builds with lobs self.assertEqual(self.call_action("releasedel", { "opsys": "fedora", "opsys-release": "24", }), 0) self.assertEqual(bosra, init_bosra) self.assertTrue(pkg_stay.has_lob("package")) self.assertFalse(pkg_del.has_lob("package")) self.assertEqual(self.db.session.query(Report).count(), 2) self.assertEqual(self.db.session.query(Problem).count(), 2) # delete release f20 with assigned bugs, reports and problems self.assertEqual(self.call_action("releasedel", { "opsys": "fedora", "opsys-release": "20", }), 0) self.assertEqual(self.db.session.query(Report).count(), 0) self.assertEqual(self.db.session.query(Problem).count(), 0) # attempt to delete deleted release f20 self.assertEqual(self.call_action("releasedel", { "opsys": "fedora", "opsys-release": "20", }), 1)