def store_rpm_provides(db: Database, package: Package, nogpgcheck: bool = False) -> None:
"""
Save RPM provides of `package` to storage.
Expects pyfaf.storage.opsys.Package object.
"""
pkg_id = package.id
ts = rpm.ts()
rpm_file = package.get_lob_fd("package")
if not rpm_file:
raise FafError("Package {0} has no lob stored".format(package.name))
if nogpgcheck:
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES) #pylint: disable=protected-access
try:
header = ts.hdrFromFdno(rpm_file.fileno())
except rpm.error as exc:
rpm_file.close()
raise FafError("rpm error: {0}".format(exc)) from exc
files = header.fiFromHeader()
log.debug("%s contains %d files", package.nvra(), len(files))
# Invalid name for type variable
# pylint: disable-msg=C0103
for f in files:
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = f[0]
new.flags = 0
db.session.add(new)
provides = header.dsFromHeader('providename')
for p in provides:
if len(p.N()) > 1024:
log.warning("Provides item in RPM header of %s longer than 1024 "
"characters. Skipping", package.name)
continue
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = p.N()
new.flags = p.Flags()
evr = p.EVR()
if evr:
try:
new.epoch, new.version, new.release = parse_evr(evr)
except ValueError as ex:
log.warning("Unparsable EVR ‘%s’ of %s in Provides of %s: %s. "
"Skipping",
evr, p.N(), package.name, ex)
continue
db.session.add(new)
rpm_file.close()
db.session.flush()
def test_problem_version_complex_filter(self):
"""
Test if filtering yields two problems
when only one of them matches release
"""
build = Build()
build.base_package_name = "kernel"
build.epoch = 0
build.version = "3.13.10"
build.release = "100.fc20"
self.db.session.add(build)
pkg = Package()
pkg.build = build
pkg.arch = self.arch_x86_64
pkg.name = "kernel"
self.db.session.add(pkg)
self.db.session.commit()
self.save_report("ureport_kerneloops2")
self.call_action("create-problems")
self.db.session.commit()
qs = "since_version=3.12&since_release=300.fc20"
r = self.app.get("/problems/?{0}".format(qs))
self.assertIn("problems/1", r.data)
self.assertIn("problems/2", r.data)
def test_problem_version_complex_filter(self):
"""
Test if filtering yields two problems
when only one of them matches release
"""
build = Build()
build.base_package_name = "kernel"
build.epoch = 0
build.version = "3.13.10"
build.release = "100.fc20"
self.db.session.add(build)
pkg = Package()
pkg.build = build
pkg.arch = self.arch_x86_64
pkg.name = "kernel"
self.db.session.add(pkg)
self.db.session.commit()
self.save_report("ureport_kerneloops2")
self.call_action("create-problems")
self.db.session.commit()
qs = "since_version=3.12&since_release=300.fc20"
r = self.app.get("/problems/?{0}".format(qs))
self.assertIn(b"problems/1", r.data)
self.assertIn(b"problems/2", r.data)
def cleanup_packages_testing(self, repo_type):
self.assign_release_to_builds_testing(repo_type)
# add package and lob
pkg = Package()
pkg.build = self.db.session.query(Build).first()
pkg.arch = self.db.session.query(Arch).first()
pkg.name = "pkg-test"
self.db.session.add(pkg)
self.db.session.flush()
config["storage.lobdir"] = "/tmp/faf_test_data/lob"
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg.save_lob("package", sample, truncate=True)
self.assertTrue(pkg.has_lob("package"))
init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(
self.call_action_ordered_args(
"cleanup-packages",
[
"Fedora", # OPSYS
"24", # RELEASE
]),
0)
bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(bosra, init_bosra - 2)
self.assertFalse(pkg.has_lob("package"))
def cleanup_packages_testing_arch(self, repo_type):
self.assign_release_to_builds_testing(repo_type)
# add package and lob
pkg = Package()
pkg.build = self.db.session.query(Build).first()
pkg.arch = self.db.session.query(Arch).first()
pkg.name = "pkg-test"
self.db.session.add(pkg)
self.db.session.flush()
config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg.save_lob("package", sample, truncate=True)
self.assertTrue(pkg.has_lob("package"))
init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(
self.call_action("cleanup-packages", {
"arch": "noarch",
}), 0)
bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(bosra, init_bosra)
self.assertEqual(
self.call_action("cleanup-packages", {
"arch": "x86_64",
}), 0)
bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(bosra, init_bosra - 2)
self.assertFalse(pkg.has_lob("package"))
def packages(self):
self.begin('Packages')
arches = self.ses.query(Arch).all()
for build in self.ses.query(Build):
for package_name in data.COMPS[build.component.name]['packages']:
new = Package()
new.arch = random.choice(arches)
new.build = build
new.name = package_name
self.add(new)
self.commit()
def packages(self):
self.begin('Packages')
arches = self.ses.query(Arch).all()
for build in self.ses.query(Build):
for package_name in data.COMPS[build.component.name]['packages']:
new = Package()
new.arch = random.choice(arches)
new.build = build
new.name = package_name
self.add(new)
self.commit()
def cleanup_packages_testing(self, repo_type):
self.assign_release_to_builds_testing(repo_type)
# add package and lob
pkg = Package()
pkg.build = self.db.session.query(Build).first()
pkg.arch = self.db.session.query(Arch).first()
pkg.name = "pkg-test"
self.db.session.add(pkg)
self.db.session.flush()
config["storage.lobdir"] = "/tmp/faf_test_data/lob"
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg.save_lob("package", sample, truncate=True)
self.assertTrue(pkg.has_lob("package"))
init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(self.call_action_ordered_args("cleanup-packages", [
"Fedora", # OPSYS
"24", # RELEASE
]), 0)
bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(bosra, init_bosra - 2)
self.assertFalse(pkg.has_lob("package"))
def test_store_rpm_deps(self):
"""
"""
# add required stuff to db
arch = Arch()
arch.name = "noarch"
self.db.session.add(arch)
build = Build()
build.base_package_name = "sample"
build.version = "1"
build.release = "1"
build.epoch = "0"
self.db.session.add(build)
pkg = Package()
pkg.name = "sample"
pkg.pkgtype = "rpm"
pkg.arch = arch
pkg.build = build
self.db.session.add(pkg)
self.db.session.flush()
# save sample rpm
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg.save_lob("package", sample, truncate=True)
# get dependencies
res = store_rpm_deps(self.db, pkg, nogpgcheck=True)
self.assertIs(res, True)
expected_deps = [
("PROVIDES", "sample", 8),
("PROVIDES", "/sample", 0),
("PROVIDES", "nothing-new", 0),
(
"REQUIRES",
"nothing",
0,
),
(
"CONFLICTS",
"surprisingly-nothing",
0,
),
]
found_deps = []
for dep in self.db.session.query(PackageDependency).all():
found_deps.append((dep.type, dep.name, dep.flags))
for dep in expected_deps:
self.assertIn(dep, found_deps)
def setUp(self):
super(ProblemsTestCase, self).setUp()
self.basic_fixtures()
build = Build()
build.base_package_name = "kernel"
build.epoch = 0
build.version = "3.12.10"
build.release = "300.fc20"
self.db.session.add(build)
pkg = Package()
pkg.build = build
pkg.arch = self.arch_x86_64
pkg.name = "kernel"
self.db.session.add(pkg)
self.db.session.commit()
self.save_report("ureport_kerneloops")
self.call_action("create-problems")
self.db.session.commit()
def setUp(self):
super(ProblemsTestCase, self).setUp()
self.basic_fixtures()
build = Build()
build.base_package_name = "kernel"
build.epoch = 0
build.version = "3.12.10"
build.release = "300.fc20"
self.db.session.add(build)
pkg = Package()
pkg.build = build
pkg.arch = self.arch_x86_64
pkg.name = "kernel"
self.db.session.add(pkg)
self.db.session.commit()
self.save_report("ureport_kerneloops")
self.call_action("create-problems")
self.db.session.commit()
def downgrade() -> None:
for (pkgid, build, pkgtype) in get_bind().execute(
sa.select(
[package.c.id,
package.c.build_id,
package.c.pkgtype]
)):
db_package = Package()
db_package.id = pkgid
db_package.build_id = build
db_package.pkgtype = pkgtype
ext_len = len(pkgtype) + 1
lobpath = db_package.get_lob_path("package")
if lobpath.endswith(".{}".format(pkgtype)) and os.path.isfile(lobpath):
os.rename(lobpath, lobpath[:-ext_len])
def downgrade() -> None:
for (pkgid, build, pkgtype) in get_bind().execute(
sa.select(
[package.c.id,
package.c.build_id,
package.c.pkgtype]
)):
db_package = Package()
db_package.id = pkgid
db_package.build_id = build
db_package.pkgtype = pkgtype
ext_len = len(pkgtype) + 1
# Package inherits get_lob_path() from GenericTable.
lobpath = db_package.get_lob_path("package") # pylint: disable=no-member
if lobpath.endswith(".{}".format(pkgtype)) and os.path.isfile(lobpath):
os.rename(lobpath, lobpath[:-ext_len])
def test_store_rpm_deps(self):
"""
"""
# add required stuff to db
arch = Arch()
arch.name = "noarch"
self.db.session.add(arch)
build = Build()
build.base_package_name = "sample"
build.version = "1"
build.release = "1"
build.epoch = "0"
self.db.session.add(build)
pkg = Package()
pkg.name = "sample"
pkg.pkgtype = "rpm"
pkg.arch = arch
pkg.build = build
self.db.session.add(pkg)
self.db.session.flush()
# save sample rpm
with open("sample_rpms/sample-1.0-1.fc18.noarch.rpm",
mode='rb') as sample:
pkg.save_lob("package", sample, truncate=True)
# get dependencies
store_rpm_deps(self.db, pkg, nogpgcheck=True)
expected_deps = [
("PROVIDES", "sample", 8),
("PROVIDES", "/sample", 0),
("PROVIDES", "nothing-new", 0),
(
"REQUIRES",
"nothing",
0,
),
(
"CONFLICTS",
"surprisingly-nothing",
0,
),
]
found_deps = []
for dep in self.db.session.query(PackageDependency).all():
found_deps.append((dep.type, dep.name, dep.flags))
for dep in expected_deps:
self.assertIn(dep, found_deps)
build = Build()
build.base_package_name = "sample-broken"
build.version = "1"
build.release = "1"
build.epoch = 0
self.db.session.add(build)
pkg = Package()
pkg.name = "sample-broken"
pkg.pkgtype = "rpm"
pkg.arch = arch
pkg.build = build
self.db.session.add(pkg)
self.db.session.flush()
with open("sample_rpms/sample-broken-1-1.el7.noarch.rpm",
mode='rb') as sample:
pkg.save_lob("package", sample, truncate=True)
with self.assertRaises(FafError):
store_rpm_deps(self.db, pkg, nogpgcheck=True)
# Safety flush
self.db.session.flush()
query = self.db.session.query(PackageDependency)
# Filter out rich RPM dependencies
dependencies = filter(lambda x: "rpmlib" not in x.name, query.all())
# The transaction was rolled back and nothing new was added
self.assertCountEqual(
map(lambda x: (x.type, x.name, x.flags), dependencies),
expected_deps)
def test_store_rpm_provides(self):
"""
"""
# add required stuff to db
arch = Arch()
arch.name = "noarch"
self.db.session.add(arch)
build = Build()
build.base_package_name = "sample"
build.version = "1"
build.release = "1"
build.epoch = "0"
self.db.session.add(build)
pkg = Package()
pkg.name = "sample"
pkg.pkgtype = "rpm"
pkg.arch = arch
pkg.build = build
self.db.session.add(pkg)
self.db.session.flush()
# save sample rpm
with open("sample_rpms/sample-1.0-1.fc18.noarch.rpm",
mode='rb') as sample:
pkg.save_lob("package", sample, truncate=True)
# get dependencies
store_rpm_provides(self.db, pkg, nogpgcheck=True)
expected_deps = [
("PROVIDES", "sample", 8),
("PROVIDES", "/sample", 0),
("PROVIDES", "nothing-new", 0),
]
found_deps = []
for dep in self.db.session.query(PackageDependency).all():
found_deps.append((dep.type, dep.name, dep.flags))
self.assertCountEqual(found_deps, expected_deps)
build = Build()
build.base_package_name = "sample-broken"
build.version = "1"
build.release = "1"
build.epoch = 0
self.db.session.add(build)
pkg = Package()
pkg.name = "sample-broken"
pkg.pkgtype = "rpm"
pkg.arch = arch
pkg.build = build
self.db.session.add(pkg)
self.db.session.flush()
with open("sample_rpms/sample-broken-1-1.fc22.noarch.rpm",
mode='rb') as rpm_file:
pkg.save_lob("package", rpm_file, truncate=True)
with self.assertLogs(level=logging.WARNING) as captured_logs:
store_rpm_provides(self.db, pkg, nogpgcheck=True)
self.assertEqual(captured_logs.output, [
"WARNING:faf.pyfaf.faf_rpm:Unparsable EVR ‘%{epoch}:1’ of "
"zabbix in Provides of sample-broken: EVR string "
"contains a non-numeric epoch: %{epoch}. Skipping"
])
# Safety flush
self.db.session.flush()
query = self.db.session.query(PackageDependency)
# Filter out rich RPM dependencies
dependencies = filter(lambda x: "rpmlib" not in x.name, query.all())
# Only provides that were correctly formatted were added
expected_deps.extend([
("PROVIDES", "sample-broken", 8),
("PROVIDES", "happiness", 0),
("PROVIDES", "joy", 0),
("PROVIDES", "love", 0),
])
self.assertCountEqual(
((x.type, x.name, x.flags) for x in dependencies), expected_deps)
build = Build()
build.base_package_name = "sample-provides-too-long"
build.version = "1"
build.release = "1"
build.epoch = 0
self.db.session.add(build)
pkg = Package()
pkg.name = "sample-provides-too-long"
pkg.pkgtype = "rpm"
pkg.arch = arch
pkg.build = build
self.db.session.add(pkg)
self.db.session.flush()
with open("sample_rpms/sample-provides-too-long-1-1.fc33.noarch.rpm",
mode='rb') as rpm_file:
pkg.save_lob("package", rpm_file, truncate=True)
with self.assertLogs(level=logging.WARNING) as captured_logs:
store_rpm_provides(self.db, pkg, nogpgcheck=True)
self.assertEqual(captured_logs.output, [
"WARNING:faf.pyfaf.faf_rpm:Provides item in RPM header of "
"sample-provides-too-long longer than 1024 characters. "
"Skipping"
])
# Safety flush
self.db.session.flush()
query = self.db.session.query(PackageDependency)
# Filter out rich RPM dependencies
dependencies = filter(lambda x: "rpmlib" not in x.name, query.all())
# Only provides that were correctly formatted were added
expected_deps.extend([
("PROVIDES", "sample-provides-too-long", 8),
("PROVIDES", "one-thing", 0),
("PROVIDES", "another-thing", 0),
("PROVIDES", "penultimate-item", 0),
("PROVIDES", "the-last-one", 0),
])
self.assertCountEqual(
((x.type, x.name, x.flags) for x in dependencies), expected_deps)
def test_get_packages_and_their_reports_unknown_packages(self):
"""
"""
# add required stuff to db
arch = Arch()
arch.name = "noarch"
self.db.session.add(arch)
arch1 = Arch()
arch1.name = "x86_64"
self.db.session.add(arch1)
build = Build()
build.base_package_name = "sample"
build.version = "1"
build.release = "1"
build.epoch = "0"
self.db.session.add(build)
pkg = Package()
pkg.name = "sample"
pkg.pkgtype = "rpm"
pkg.arch = arch
pkg.build = build
self.db.session.add(pkg)
#print(pkg.nevra())
#different arch
pkg2 = Package()
pkg2.name = "sample"
pkg2.pkgtype = "rpm"
pkg2.arch = arch1
pkg2.build = build
self.db.session.add(pkg2)
#print(pkg2.nevra())
build2 = Build()
build2.base_package_name = "sample"
build2.version = "1"
build2.release = "2"
build2.epoch = "0"
self.db.session.add(build2)
pkg3 = Package()
pkg3.name = "sample"
pkg3.pkgtype = "rpm"
pkg3.arch = arch
pkg3.build = build2
self.db.session.add(pkg3)
#print(pkg3.nevra())
problem = Problem()
self.db.session.add(problem)
opsys = OpSys()
opsys.name = "Fedora"
self.db.session.add(opsys)
opsys_component = OpSysComponent()
opsys_component.name = "core"
opsys_component.opsys = opsys
self.db.session.add(opsys_component)
report = Report()
report.type = "type"
report.count = 2
report.problem = problem
report.component = opsys_component
self.db.session.add(report)
report_unknown = ReportUnknownPackage()
report_unknown.report = report
report_unknown.name = pkg.name
report_unknown.epoch = pkg.build.epoch
report_unknown.version = pkg.build.version
report_unknown.release = pkg.build.release
report_unknown.arch = pkg.arch
report_unknown.count = 1
self.db.session.add(report_unknown)
report_unknown2 = ReportUnknownPackage()
report_unknown2.report = report
report_unknown2.name = pkg2.name
report_unknown2.epoch = pkg2.build.epoch
report_unknown2.version = pkg2.build.version
report_unknown2.release = pkg2.build.release
report_unknown2.arch = pkg2.arch
report_unknown2.count = 1
self.db.session.add(report_unknown2)
report_unknown3 = ReportUnknownPackage()
report_unknown3.report = report
report_unknown3.name = "nonsense"
report_unknown3.epoch = pkg.build.epoch
report_unknown3.version = pkg.build.version
report_unknown3.release = pkg.build.release
report_unknown3.arch = pkg.arch
report_unknown3.count = 1
self.db.session.add(report_unknown3)
self.db.session.flush()
packages_and_their_reports_unknown_packages = \
get_packages_and_their_reports_unknown_packages(self.db).all()
self.assertEqual(len(packages_and_their_reports_unknown_packages), 2)
self.assertIn(
(pkg, report_unknown), packages_and_their_reports_unknown_packages)
self.assertIn(
(pkg2, report_unknown2), packages_and_their_reports_unknown_packages)
def test_get_packages_and_their_reports_unknown_packages(self):
"""
"""
# add required stuff to db
arch = Arch()
arch.name = "noarch"
self.db.session.add(arch)
arch1 = Arch()
arch1.name = "x86_64"
self.db.session.add(arch1)
build = Build()
build.base_package_name = "sample"
build.version = "1"
build.release = "1"
build.epoch = "0"
self.db.session.add(build)
pkg = Package()
pkg.name = "sample"
pkg.pkgtype = "rpm"
pkg.arch = arch
pkg.build = build
self.db.session.add(pkg)
#print(pkg.nevra())
#different arch
pkg2 = Package()
pkg2.name = "sample"
pkg2.pkgtype = "rpm"
pkg2.arch = arch1
pkg2.build = build
self.db.session.add(pkg2)
#print(pkg2.nevra())
build2 = Build()
build2.base_package_name = "sample"
build2.version = "1"
build2.release = "2"
build2.epoch = "0"
self.db.session.add(build2)
pkg3 = Package()
pkg3.name = "sample"
pkg3.pkgtype = "rpm"
pkg3.arch = arch
pkg3.build = build2
self.db.session.add(pkg3)
#print(pkg3.nevra())
problem = Problem()
self.db.session.add(problem)
opsys = OpSys()
opsys.name = "Fedora"
self.db.session.add(opsys)
opsys_component = OpSysComponent()
opsys_component.name = "core"
opsys_component.opsys = opsys
self.db.session.add(opsys_component)
report = Report()
report.type = "type"
report.count = 2
report.problem = problem
report.component = opsys_component
self.db.session.add(report)
report_unknown = ReportUnknownPackage()
report_unknown.report = report
report_unknown.name = pkg.name
report_unknown.epoch = pkg.build.epoch
report_unknown.version = pkg.build.version
report_unknown.release = pkg.build.release
report_unknown.arch = pkg.arch
report_unknown.count = 1
self.db.session.add(report_unknown)
report_unknown2 = ReportUnknownPackage()
report_unknown2.report = report
report_unknown2.name = pkg2.name
report_unknown2.epoch = pkg2.build.epoch
report_unknown2.version = pkg2.build.version
report_unknown2.release = pkg2.build.release
report_unknown2.arch = pkg2.arch
report_unknown2.count = 1
self.db.session.add(report_unknown2)
report_unknown3 = ReportUnknownPackage()
report_unknown3.report = report
report_unknown3.name = "nonsense"
report_unknown3.epoch = pkg.build.epoch
report_unknown3.version = pkg.build.version
report_unknown3.release = pkg.build.release
report_unknown3.arch = pkg.arch
report_unknown3.count = 1
self.db.session.add(report_unknown3)
self.db.session.flush()
packages_and_their_reports_unknown_packages = \
get_packages_and_their_reports_unknown_packages(self.db).all()
self.assertEqual(len(packages_and_their_reports_unknown_packages), 2)
self.assertIn((pkg, report_unknown),
packages_and_their_reports_unknown_packages)
self.assertIn((pkg2, report_unknown2),
packages_and_their_reports_unknown_packages)
def cleanup_unassigned_testing(self, repo_type):
self.assign_release_to_builds_testing(repo_type)
# add package and lob that will not be deleted
pkg_stay = Package()
pkg_stay.build = self.db.session.query(Build).first()
pkg_stay.arch = self.db.session.query(Arch).first()
pkg_stay.name = "pkg-test-stay"
self.db.session.add(pkg_stay)
self.db.session.flush()
config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg_stay.save_lob("package", sample, truncate=True)
self.assertTrue(pkg_stay.has_lob("package"))
# add build and package and lob that will be deleted
build = Build()
build.base_package_name = "build_unassigned"
build.epoch = 0
build.version = "1.2.3"
build.release = "20.fc23"
self.db.session.add(build)
pkg_del = Package()
pkg_del.build = build
pkg_del.arch = self.db.session.query(Arch).first()
pkg_del.name = "pkg-test-del"
self.db.session.add(pkg_del)
self.db.session.flush()
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg_del.save_lob("package", sample, truncate=True)
self.assertTrue(pkg_del.has_lob("package"))
init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(
self.call_action_ordered_args("cleanup-unassigned", ["--force"]),
0)
bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(bosra, init_bosra)
self.assertFalse(pkg_del.has_lob("package"))
self.assertTrue(pkg_stay.has_lob("package"))
def releasedel_testing(self, repo_type):
self.assertEqual(
self.call_action("releasedel", {
"opsys": "fedora",
"opsys-release": "1337",
}), 1)
# add release f24 and assign two builds
self.assign_release_to_builds_testing(repo_type)
# assign sample_repo to f24 release (sample_repo comes from ^^^)
self.assertEqual(
self.call_action_ordered_args(
"repoassign",
[
"sample_repo", # NAME
"Fedora 24", # OPSYS
"x86_64", # ARCH
]),
0)
tracker = Bugtracker(name="fedora-bugzilla")
self.db.session.add(tracker)
self.save_report('ureport_f20')
self.save_report('ureport_core')
self.call_action("create-problems")
bzuser = BzUser(name="Fake user",
email="*****@*****.**",
real_name="Fake name",
can_login=False)
self.db.session.add(bzuser)
bug = BzBug()
bug.id = 123456
bug.summary = "Fake bug"
bug.status = "NEW"
bug.creation_time = datetime.now()
bug.last_change_time = datetime.now()
bug.whiteboard = "empty"
bug.tracker = tracker
bug.creator = bzuser
bug.component = self.comp_faf
bug.opsysrelease = self.release_20
bug.private = False
self.db.session.add(bug)
# add package and lob that will be deleted
pkg_del = Package()
pkg_del.build = self.db.session.query(Build).first()
pkg_del.arch = self.db.session.query(Arch).first()
pkg_del.name = "pkg-test-del"
self.db.session.add(pkg_del)
self.db.session.flush()
config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg_del.save_lob("package", sample, truncate=True)
self.assertTrue(pkg_del.has_lob("package"))
# add build and package and lob that will not be deleted
build = Build()
build.base_package_name = "build_unassigned"
build.epoch = 0
build.version = "1.2.3"
build.release = "20.fc23"
self.db.session.add(build)
pkg_stay = Package()
pkg_stay.build = build
pkg_stay.arch = self.db.session.query(Arch).first()
pkg_stay.name = "pkg-test-stay"
self.db.session.add(pkg_stay)
self.db.session.flush()
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg_stay.save_lob("package", sample, truncate=True)
self.assertTrue(pkg_stay.has_lob("package"))
init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
bosra = self.db.session.query(BuildOpSysReleaseArch).count()
# delete release f24 with assigned repos and builds with lobs
self.assertEqual(
self.call_action("releasedel", {
"opsys": "fedora",
"opsys-release": "24",
}), 0)
self.assertEqual(bosra, init_bosra)
self.assertTrue(pkg_stay.has_lob("package"))
self.assertFalse(pkg_del.has_lob("package"))
self.assertEqual(self.db.session.query(Report).count(), 2)
self.assertEqual(self.db.session.query(Problem).count(), 2)
# delete release f20 with assigned bugs, reports and problems
self.assertEqual(
self.call_action("releasedel", {
"opsys": "fedora",
"opsys-release": "20",
}), 0)
self.assertEqual(self.db.session.query(Report).count(), 0)
self.assertEqual(self.db.session.query(Problem).count(), 0)
# attempt to delete deleted release f20
self.assertEqual(
self.call_action("releasedel", {
"opsys": "fedora",
"opsys-release": "20",
}), 1)
def run(self, cmdline, db) -> None:
repo_instances = []
for repo in db.session.query(Repo):
if cmdline.REPO and repo.name not in cmdline.REPO:
continue
if cmdline.match_repos not in repo.name:
continue
if not repo.type in repo_types:
self.log_error("No plugin installed to handle repository type "
"{0}, skipping.".format(repo.type))
continue
if repo.opsys_list: # parametrized
self.log_info("Processing parametrized repo '{0}'".format(
repo.name))
if not repo.arch_list:
self.log_error("Parametrized repository is not assigned"
" with an architecture, skipping")
continue
try:
repo_instances += list(
self._get_parametrized_variants(repo))
except: # pylint: disable=bare-except
self.log_error(
"No valid mirror for repository '{0}', skipping".
format(repo.name))
continue
elif repo.opsysrelease_list:
self.log_info(
"Processing repo '{0}' assigned with OpSysRelease".format(
repo.name))
if not repo.arch_list:
self.log_error("OpSysRelease repository is not assigned"
" with an architecture, skipping")
continue
try:
repo_instances += list(
self._get_opsysrelease_variants(repo))
except: # pylint: disable=bare-except
self.log_error(
"No valid mirror for repository '{0}', skipping".
format(repo.name))
continue
else:
if any('$' in url.url for url in repo.url_list):
self.log_error("No operating system assigned to"
"parametrized repo '{0}', skipping".format(
repo.name))
continue
for arch in repo.arch_list:
try:
repo_instance = {
'instance':
repo_types[repo.type](
repo.name, [url.url for url in repo.url_list]),
'opsys':
None,
'release':
None,
'arch':
arch.name,
'nogpgcheck':
repo.nogpgcheck
}
repo_instances.append(repo_instance)
except: # pylint: disable=bare-except
self.log_error(
"No valid mirror for repository '{0}', skipping".
format(repo.name))
continue
cmdline.name_prefix = cmdline.name_prefix.lower()
architectures = dict((x.name, x) for x in get_archs(db))
for repo_instance in repo_instances:
self.log_info(
"Processing repository '{0}' assigned to OS release '{1}' and arch '{2}', URL: '{3}'"
.format(repo_instance['instance'].name,
repo_instance['release'], repo_instance['arch'],
repo_instance['instance'].urls))
if cmdline.no_cache:
repo_instance['instance'].cache_lifetime = 0
pkglist = \
repo_instance['instance'].list_packages(list(architectures.keys()))
total = len(pkglist)
self.log_info("Repository has {0} packages".format(total))
for num, pkg in enumerate(pkglist, start=1):
self.log_info("[%d / %d] Processing package %s", num, total,
pkg["name"])
if not pkg["name"].lower().startswith(cmdline.name_prefix):
self.log_debug("Skipped package %s", pkg["name"])
continue
arch = architectures.get(pkg["arch"], None)
if not arch:
self.log_error(
"Architecture '{0}' not found, skipping".format(
pkg["arch"]))
continue
repo_arch = architectures.get(repo_instance["arch"], None)
if not repo_arch:
self.log_error(
"Architecture '{0}' not found, skipping".format(
repo_instance["arch"]))
continue
build = (db.session.query(Build).filter(
Build.base_package_name == pkg["base_package_name"]).
filter(Build.version == pkg["version"]).filter(
Build.release == pkg["release"]).filter(
Build.epoch == pkg["epoch"]).first())
if not build:
self.log_debug("Adding build %s-%s",
pkg["base_package_name"], pkg["version"])
build = Build()
build.base_package_name = pkg["base_package_name"]
build.version = pkg["version"]
build.release = pkg["release"]
build.epoch = pkg["epoch"]
db.session.add(build)
db.session.flush()
build_arch = (db.session.query(BuildArch).filter(
BuildArch.build_id == build.id).filter(
BuildArch.arch_id == arch.id).first())
if not build_arch:
build_arch = BuildArch()
build_arch.build = build
build_arch.arch = arch
db.session.add(build_arch)
db.session.flush()
build_opsysrelease_arch = (db.session.query(
BuildOpSysReleaseArch
).join(Build).join(OpSysRelease).join(Arch).filter(
Build.id == build.id).filter(
OpSys.name == repo_instance['opsys']).filter(
OpSysRelease.version == repo_instance['release']
).filter(Arch.name == repo_instance['arch']).first())
if not build_opsysrelease_arch and repo_instance[
'release'] and repo_instance['opsys']:
self.log_info("Adding link between build '{0}-{1}' "
"and operating system '{2} {3} {4}'".format(
pkg["base_package_name"], pkg["version"],
repo_instance['opsys'],
repo_instance['release'],
repo_instance['arch']))
opsysrelease = (db.session.query(OpSysRelease).filter(
OpSys.name == repo_instance['opsys']).filter(
OpSysRelease.version ==
repo_instance['release']).first())
bosra = BuildOpSysReleaseArch()
bosra.build = build
bosra.opsysrelease = opsysrelease
bosra.arch = repo_arch
db.session.add(bosra)
db.session.flush()
package = (db.session.query(Package).filter(
Package.name == pkg["name"]).filter(
Package.pkgtype == pkg["type"]).filter(
Package.build == build).filter(
Package.arch == arch).first())
if not package:
self.log_info("Adding package {0}".format(pkg["filename"]))
package = Package()
package.name = pkg["name"]
package.pkgtype = pkg["type"]
package.arch = arch
package.build = build
db.session.add(package)
db.session.flush()
if cmdline.no_download_rpm:
continue
# Catching too general exception Exception
# pylint: disable-msg=W0703
try:
self.log_info("Downloading {0}".format(pkg["url"]))
self._download(package, "package", pkg["url"])
except Exception as exc:
self.log_error(
"Exception ({0}) after multiple attempts"
" while trying to download {1},"
" skipping.".format(exc, pkg["url"]))
db.session.delete(package)
db.session.flush()
continue
# pylint: enable-msg=W0703
if pkg["type"] == "rpm":
try:
store_rpm_provides(db, package,
repo_instance['nogpgcheck'])
except FafError as ex:
self.log_error(
"Post-processing failed, skipping: {}".format(
ex))
db.session.delete(package)
db.session.flush()
continue
if cmdline.no_store_rpm:
try:
package.del_lob("package")
self.log_info("Package deleted.")
except Exception as exc: # pylint: disable=broad-except
self.log_error("Error deleting the RPM file.")
else:
self.log_debug("Known package %s", pkg["filename"])
if not package.has_lob(
"package") and not cmdline.no_download_rpm:
self.log_info(
"Package {} does not have a LOB. Re-downloading.".
format(pkg["name"]))
try:
self._download(package, "package", pkg["url"])
except (FafError, URLError) as exc:
self.log_error(
"Exception ({0}) after multiple attempts"
" while trying to download {1},"
" skipping.".format(exc, pkg["url"]))
def setUp(self):
super(MPFTestCase, self).setUp()
self.basic_fixtures()
# Fixed problem 1
problem = Problem()
self.db.session.add(problem)
report = Report()
report.type = "type"
report.count = 2
report.problem = problem
report.component = self.comp_systemd
self.db.session.add(report)
with open("sample_reports/ureport_systemd2", "r") as file:
self.ureport_systemd2 = json.load(file)
with open("sample_reports/ureport_systemd77", "r") as file:
self.ureport_systemd77 = json.load(file)
problemplugin = problemtypes[self.ureport_systemd2["problem"]["type"]]
report_hash = problemplugin.hash_ureport(
self.ureport_systemd2["problem"])
db_hash = ReportHash()
db_hash.hash = report_hash
db_hash.report = report
self.db.session.add(db_hash)
report_unknown = ReportUnknownPackage()
report_unknown.report = report
report_unknown.name = "systemd-test"
report_unknown.epoch = 0
report_unknown.version = 2
report_unknown.release = 1
report_unknown.semver = "2.0.0"
report_unknown.semrel = "1.0.0"
report_unknown.arch = self.arch_i686
report_unknown.count = 1
report_unknown.type = "CRASHED"
self.db.session.add(report_unknown)
report_unknown = ReportUnknownPackage()
report_unknown.report = report
report_unknown.name = "faf"
report_unknown.epoch = 0
report_unknown.version = 1
report_unknown.release = 1
report_unknown.semver = "1.0.0"
report_unknown.semrel = "1.0.0"
report_unknown.arch = self.arch_i686
report_unknown.count = 1
report_unknown.type = "RELATED"
self.db.session.add(report_unknown)
report_osr = ReportOpSysRelease()
report_osr.count = 77
report_osr.opsysrelease = self.release_20
report_osr.report = report
self.db.session.add(report_osr)
report_osr = ReportOpSysRelease()
report_osr.count = 77
report_osr.opsysrelease = self.release_19
report_osr.report = report
self.db.session.add(report_osr)
self.problem_fixed1 = problem
# Fixed problem 2
problem = Problem()
self.db.session.add(problem)
report = Report()
report.type = "type"
report.count = 2
report.problem = problem
report.component = self.comp_faf
self.db.session.add(report)
build = Build()
build.base_package_name = "faf"
build.epoch = 0
build.version = "3.4"
build.release = "5"
self.db.session.add(build)
pkg = Package()
pkg.build = build
pkg.arch = self.arch_i686
pkg.name = "faf-test"
self.db.session.add(pkg)
rpkg = ReportPackage()
rpkg.report = report
rpkg.type = "CRASHED"
rpkg.installed_package = pkg
rpkg.count = 66
self.db.session.add(rpkg)
report_osr = ReportOpSysRelease()
report_osr.count = 77
report_osr.opsysrelease = self.release_20
report_osr.report = report
self.db.session.add(report_osr)
self.problem_fixed2 = problem
# Unfixed problem 1
problem = Problem()
self.db.session.add(problem)
report = Report()
report.type = "type"
report.count = 2
report.problem = problem
report.component = self.comp_systemd
self.db.session.add(report)
report_unknown = ReportUnknownPackage()
report_unknown.report = report
report_unknown.name = "systemd"
report_unknown.epoch = 0
report_unknown.version = 3
report_unknown.release = 1
report_unknown.semver = "3.0.0"
report_unknown.semrel = "1.0.0"
report_unknown.arch = self.arch_i686
report_unknown.count = 1
report_unknown.type = "CRASHED"
self.db.session.add(report_unknown)
report_osr = ReportOpSysRelease()
report_osr.count = 77
report_osr.opsysrelease = self.release_20
report_osr.report = report
self.db.session.add(report_osr)
self.problem_unfixed1 = problem
# Unfixed problem 2
problem = Problem()
self.db.session.add(problem)
report = Report()
report.type = "type"
report.count = 2
report.problem = problem
report.component = self.comp_systemd
self.db.session.add(report)
report_unknown = ReportUnknownPackage()
report_unknown.report = report
report_unknown.name = "systemd"
report_unknown.epoch = 0
report_unknown.version = 3
report_unknown.release = 0
report_unknown.semver = "3.0.0"
report_unknown.semrel = "0.0.0"
report_unknown.arch = self.arch_i686
report_unknown.count = 1
report_unknown.type = "CRASHED"
self.db.session.add(report_unknown)
report_osr = ReportOpSysRelease()
report_osr.count = 77
report_osr.opsysrelease = self.release_20
report_osr.report = report
self.db.session.add(report_osr)
self.problem_unfixed2 = problem
# Unfixed problem 3
problem = Problem()
self.db.session.add(problem)
report = Report()
report.type = "type"
report.count = 2
report.problem = problem
report.component = self.comp_faf
self.db.session.add(report)
build = Build()
build.base_package_name = "faf"
build.epoch = 0
build.version = "3.110"
build.release = "5"
self.db.session.add(build)
pkg = Package()
pkg.build = build
pkg.arch = self.arch_i686
pkg.name = "faf-test"
self.db.session.add(pkg)
rpkg = ReportPackage()
rpkg.report = report
rpkg.type = "CRASHED"
rpkg.installed_package = pkg
rpkg.count = 66
self.db.session.add(rpkg)
report_osr = ReportOpSysRelease()
report_osr.count = 77
report_osr.opsysrelease = self.release_20
report_osr.report = report
self.db.session.add(report_osr)
self.problem_unfixed3 = problem
self.db.session.flush()
systems['fedora'].get_released_builds = get_released_builds_mock
systems['fedora'].build_aging_days = 7
def run(self, cmdline, db):
repo_instances = []
for repo in db.session.query(Repo):
if cmdline.NAME and repo.name not in cmdline.NAME:
continue
if not repo.type in repo_types:
self.log_error("No plugin installed to handle repository type "
"{0}, skipping.".format(repo.type))
continue
if "$" in repo.url: # parametrized
self.log_info("Processing parametrized repo '{0}'"
.format(repo.name))
if not repo.opsys_list:
self.log_error("Parametrized repository is not assigned"
" with an operating system")
return 1
if not repo.arch_list:
self.log_error("Parametrized repository is not assigned"
" with an architecture")
return 1
repo_instances += list(self._get_parametrized_variants(repo))
else:
repo_instance = repo_types[repo.type](repo.name, repo.url)
repo_instances.append(repo_instance)
architectures = map(lambda x: x.name, db.session.query(Arch))
for repo_instance in repo_instances:
self.log_info("Processing repository '{0}' URL: '{1}'"
.format(repo_instance.name, repo_instance.urls[0]))
pkglist = repo_instance.list_packages(architectures)
total = len(pkglist)
self.log_info("Repository has '{0}' packages".format(total))
for num, pkg in enumerate(pkglist):
self.log_debug("[{0} / {1}] Processing package {2}"
.format(num + 1, total, pkg["name"]))
arch = get_arch_by_name(db, pkg["arch"])
if not arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(pkg["arch"]))
continue
build = (db.session.query(Build)
.filter(Build.base_package_name ==
pkg["base_package_name"])
.filter(Build.version == pkg["version"])
.filter(Build.release == pkg["release"])
.filter(Build.epoch == pkg["epoch"])
.first())
if not build:
self.log_debug("Adding build {0}-{1}".format(
pkg["base_package_name"], pkg["version"]))
build = Build()
build.base_package_name = pkg["base_package_name"]
build.version = pkg["version"]
build.release = pkg["release"]
build.epoch = pkg["epoch"]
db.session.add(build)
build_arch = BuildArch()
build_arch.build = build
build_arch.arch = arch
db.session.add(build_arch)
db.session.flush()
package = (db.session.query(Package)
.filter(Package.name == pkg["name"])
.filter(Package.pkgtype == pkg["type"])
.filter(Package.build == build)
.filter(Package.arch == arch)
.first())
if not package:
self.log_info("Adding package {0}".format(pkg["filename"]))
package = Package()
package.name = pkg["name"]
package.pkgtype = pkg["type"]
package.arch = arch
package.build = build
db.session.add(package)
db.session.flush()
if cmdline.no_download_rpm:
continue
# Catching too general exception Exception
# pylint: disable-msg=W0703
try:
self.log_info("Downloading {0}".format(pkg["url"]))
self._download(package, "package", pkg["url"])
except Exception as exc:
self.log_error("Exception ({0}) after multiple attemps"
" while trying to download {1},"
" skipping.".format(exc, pkg["url"]))
db.session.delete(package)
db.session.flush()
continue
# pylint: enable-msg=W0703
res = True
if pkg["type"] == "rpm":
res = store_rpm_deps(db, package, repo.nogpgcheck)
if not res:
self.log_error("Post-processing failed, skipping")
db.session.delete(package)
db.session.flush()
continue
if cmdline.no_store_rpm:
try:
package.del_lob("package")
self.log_info("Package deleted.")
except Exception as exc:
self.log_error("Error deleting the RPM file.")
else:
self.log_debug("Known package {0}".format(pkg["filename"]))
def run(self, cmdline, db):
repo_instances = []
for repo in db.session.query(Repo):
if cmdline.NAME and repo.name not in cmdline.NAME:
continue
if not repo.type in repo_types:
self.log_error("No plugin installed to handle repository type "
"{0}, skipping.".format(repo.type))
continue
if repo.opsys_list: # parametrized
self.log_info("Processing parametrized repo '{0}'"
.format(repo.name))
if not repo.arch_list:
self.log_error("Parametrized repository is not assigned"
" with an architecture")
return 1
try:
repo_instances += list(self._get_parametrized_variants(repo))
except:
self.log_error("No valid mirror for repository {0}"
.format(repo.name))
return 1
elif repo.opsysrelease_list:
self.log_info("Processing repo '{0}' assigned with OpSysRelease"
.format(repo.name))
if not repo.arch_list:
self.log_error("OpSysRelease repository is not assigned"
" with an architecture")
return 1
try:
repo_instances += list(self._get_opsysrelease_variants(repo))
except:
self.log_error("No valid mirror for repository {0}"
.format(repo.name))
return 1
else:
if any('$' in url.url for url in repo.url_list):
self.log_error("No operating system assigned to"
"parametrized repo '{0}".format(repo.name))
return 1
for arch in repo.arch_list:
try:
repo_instance = {
'instance' : repo_types[repo.type](repo.name,
[url.url for url in repo.url_list]),
'opsys' : None,
'release' : None,
'arch' : arch.name}
repo_instances.append(repo_instance)
except:
self.log_error("No valid mirror for repository {0}"
.format(repo.name))
return 1
cmdline.name_prefix = cmdline.name_prefix.lower()
architectures = dict((x.name, x) for x in get_archs(db))
for repo_instance in repo_instances:
self.log_info("Processing repository '{0}' URL: '{1}'"
.format(repo_instance['instance'].name,
repo_instance['instance'].urls))
pkglist = \
repo_instance['instance'].list_packages(architectures.keys())
total = len(pkglist)
self.log_info("Repository has '{0}' packages".format(total))
for num, pkg in enumerate(pkglist):
self.log_debug("[{0} / {1}] Processing package {2}"
.format(num + 1, total, pkg["name"]))
if not pkg["name"].lower().startswith(cmdline.name_prefix):
self.log_debug("Skipped package {0}"
.format(pkg["name"]))
continue
arch = architectures.get(pkg["arch"], None)
if not arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(pkg["arch"]))
continue
repo_arch = architectures.get(repo_instance["arch"], None)
if not repo_arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(repo_instance["arch"]))
continue
build = (db.session.query(Build)
.filter(Build.base_package_name ==
pkg["base_package_name"])
.filter(Build.version == pkg["version"])
.filter(Build.release == pkg["release"])
.filter(Build.epoch == pkg["epoch"])
.first())
if not build:
self.log_debug("Adding build {0}-{1}".format(
pkg["base_package_name"], pkg["version"]))
build = Build()
build.base_package_name = pkg["base_package_name"]
build.version = pkg["version"]
build.release = pkg["release"]
build.epoch = pkg["epoch"]
db.session.add(build)
db.session.flush()
build_arch = (db.session.query(BuildArch)
.filter(BuildArch.build_id == build.id)
.filter(BuildArch.arch_id == arch.id)
.first())
if not build_arch:
build_arch = BuildArch()
build_arch.build = build
build_arch.arch = arch
db.session.add(build_arch)
db.session.flush()
build_opsysrelease_arch = (
db.session.query(BuildOpSysReleaseArch)
.join(Build)
.join(OpSysRelease)
.join(Arch)
.filter(Build.id == build.id)
.filter(OpSys.name == repo_instance['opsys'])
.filter(OpSysRelease.version == repo_instance['release'])
.filter(Arch.name == repo_instance['arch'])
.first())
if not build_opsysrelease_arch and repo_instance['release'] and repo_instance['opsys']:
self.log_info("Adding link between build {0}-{1} "
"operating system '{2}', release '{3} and "
"architecture {4}".format(pkg["base_package_name"],
pkg["version"], repo_instance['opsys'],
repo_instance['release'], repo_instance['arch']))
opsysrelease = (
db.session.query(OpSysRelease)
.filter(OpSys.name == repo_instance['opsys'])
.filter(OpSysRelease.version == repo_instance['release'])
.first())
bosra = BuildOpSysReleaseArch()
bosra.build = build
bosra.opsysrelease = opsysrelease
bosra.arch = repo_arch
db.session.add(bosra)
db.session.flush()
package = (db.session.query(Package)
.filter(Package.name == pkg["name"])
.filter(Package.pkgtype == pkg["type"])
.filter(Package.build == build)
.filter(Package.arch == arch)
.first())
if not package:
self.log_info("Adding package {0}".format(pkg["filename"]))
package = Package()
package.name = pkg["name"]
package.pkgtype = pkg["type"]
package.arch = arch
package.build = build
db.session.add(package)
db.session.flush()
if cmdline.no_download_rpm:
continue
# Catching too general exception Exception
# pylint: disable-msg=W0703
try:
self.log_info("Downloading {0}".format(pkg["url"]))
self._download(package, "package", pkg["url"])
except Exception as exc:
self.log_error("Exception ({0}) after multiple attemps"
" while trying to download {1},"
" skipping.".format(exc, pkg["url"]))
db.session.delete(package)
db.session.flush()
continue
# pylint: enable-msg=W0703
res = True
if pkg["type"] == "rpm":
res = store_rpm_deps(db, package, repo.nogpgcheck)
if not res:
self.log_error("Post-processing failed, skipping")
db.session.delete(package)
db.session.flush()
continue
if cmdline.no_store_rpm:
try:
package.del_lob("package")
self.log_info("Package deleted.")
except Exception as exc:
self.log_error("Error deleting the RPM file.")
else:
self.log_debug("Known package {0}".format(pkg["filename"]))
def cleanup_unassigned_testing(self, repo_type):
self.assign_release_to_builds_testing(repo_type)
# add package and lob that will not be deleted
pkg_stay = Package()
pkg_stay.build = self.db.session.query(Build).first()
pkg_stay.arch = self.db.session.query(Arch).first()
pkg_stay.name = "pkg-test-stay"
self.db.session.add(pkg_stay)
self.db.session.flush()
config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg_stay.save_lob("package", sample, truncate=True)
self.assertTrue(pkg_stay.has_lob("package"))
# add build and package and lob that will be deleted
build = Build()
build.base_package_name = "build_unassigned"
build.epoch = 0
build.version = "1.2.3"
build.release = "20.fc23"
self.db.session.add(build)
pkg_del = Package()
pkg_del.build = build
pkg_del.arch = self.db.session.query(Arch).first()
pkg_del.name = "pkg-test-del"
self.db.session.add(pkg_del)
self.db.session.flush()
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg_del.save_lob("package", sample, truncate=True)
self.assertTrue(pkg_del.has_lob("package"))
init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(self.call_action_ordered_args("cleanup-unassigned", [
"--force"
]), 0)
bosra = self.db.session.query(BuildOpSysReleaseArch).count()
self.assertEqual(bosra, init_bosra)
self.assertFalse(pkg_del.has_lob("package"))
self.assertTrue(pkg_stay.has_lob("package"))
def run(self, cmdline, db):
repo_instances = []
for repo in db.session.query(Repo):
if cmdline.NAME and repo.name not in cmdline.NAME:
continue
if not repo.type in repo_types:
self.log_error("No plugin installed to handle repository type "
"{0}, skipping.".format(repo.type))
continue
if "$" in repo.url: # parametrized
self.log_info("Processing parametrized repo '{0}'"
.format(repo.name))
if not repo.opsys_list:
self.log_error("Parametrized repository is not assigned"
" with an operating system")
return 1
if not repo.arch_list:
self.log_error("Parametrized repository is not assigned"
" with an architecture")
return 1
repo_instances += list(self._get_parametrized_variants(repo))
else:
repo_instance = repo_types[repo.type](repo.name, repo.url)
repo_instances.append(repo_instance)
cmdline.name_prefix = cmdline.name_prefix.lower()
architectures = dict((x.name, x) for x in get_archs(db))
for repo_instance in repo_instances:
self.log_info("Processing repository '{0}' URL: '{1}'"
.format(repo_instance.name, repo_instance.urls[0]))
pkglist = repo_instance.list_packages(architectures.keys())
total = len(pkglist)
self.log_info("Repository has '{0}' packages".format(total))
for num, pkg in enumerate(pkglist):
self.log_debug("[{0} / {1}] Processing package {2}"
.format(num + 1, total, pkg["name"]))
if not pkg["name"].lower().startswith(cmdline.name_prefix):
self.log_debug("Skipped package {0}"
.format(pkg["name"]))
continue
arch = architectures.get(pkg["arch"], None)
if not arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(pkg["arch"]))
continue
build = (db.session.query(Build)
.filter(Build.base_package_name ==
pkg["base_package_name"])
.filter(Build.version == pkg["version"])
.filter(Build.release == pkg["release"])
.filter(Build.epoch == pkg["epoch"])
.first())
if not build:
self.log_debug("Adding build {0}-{1}".format(
pkg["base_package_name"], pkg["version"]))
build = Build()
build.base_package_name = pkg["base_package_name"]
build.version = pkg["version"]
build.release = pkg["release"]
build.epoch = pkg["epoch"]
db.session.add(build)
build_arch = BuildArch()
build_arch.build = build
build_arch.arch = arch
db.session.add(build_arch)
db.session.flush()
package = (db.session.query(Package)
.filter(Package.name == pkg["name"])
.filter(Package.pkgtype == pkg["type"])
.filter(Package.build == build)
.filter(Package.arch == arch)
.first())
if not package:
self.log_info("Adding package {0}".format(pkg["filename"]))
package = Package()
package.name = pkg["name"]
package.pkgtype = pkg["type"]
package.arch = arch
package.build = build
db.session.add(package)
db.session.flush()
if cmdline.no_download_rpm:
continue
# Catching too general exception Exception
# pylint: disable-msg=W0703
try:
self.log_info("Downloading {0}".format(pkg["url"]))
self._download(package, "package", pkg["url"])
except Exception as exc:
self.log_error("Exception ({0}) after multiple attemps"
" while trying to download {1},"
" skipping.".format(exc, pkg["url"]))
db.session.delete(package)
db.session.flush()
continue
# pylint: enable-msg=W0703
res = True
if pkg["type"] == "rpm":
res = store_rpm_deps(db, package, repo.nogpgcheck)
if not res:
self.log_error("Post-processing failed, skipping")
db.session.delete(package)
db.session.flush()
continue
if cmdline.no_store_rpm:
try:
package.del_lob("package")
self.log_info("Package deleted.")
except Exception as exc:
self.log_error("Error deleting the RPM file.")
else:
self.log_debug("Known package {0}".format(pkg["filename"]))
def releasedel_testing(self, repo_type):
self.assertEqual(self.call_action("releasedel"), 1)
self.assertEqual(self.call_action("releasedel", {
"opsys": "FooBarOS",
}), 1)
self.assertEqual(self.call_action("releasedel", {
"opsys": "fedora",
"opsys-release": "1337",
}), 1)
# add release f24 and assign two builds
self.assign_release_to_builds_testing(repo_type)
# assign sample_repo to f24 release (sample_repo comes from ^^^)
self.assertEqual(self.call_action_ordered_args("repoassign", [
"sample_repo", # NAME
"Fedora 24", # OPSYS
"x86_64", # ARCH
]), 0)
tracker = Bugtracker(name="fedora-bugzilla")
self.db.session.add(tracker)
self.save_report('ureport_f20')
self.save_report('ureport_core')
self.call_action("create-problems")
bzuser = BzUser(name="Fake user",
email="*****@*****.**",
real_name="Fake name",
can_login=False)
self.db.session.add(bzuser)
bug = BzBug()
bug.id = 123456
bug.summary = "Fake bug"
bug.status = "NEW"
bug.creation_time = datetime.now()
bug.last_change_time = datetime.now()
bug.whiteboard = "empty"
bug.tracker = tracker
bug.creator = bzuser
bug.component = self.comp_faf
bug.opsysrelease = self.release_20
bug.private = False
self.db.session.add(bug)
# add package and lob that will be deleted
pkg_del = Package()
pkg_del.build = self.db.session.query(Build).first()
pkg_del.arch = self.db.session.query(Arch).first()
pkg_del.name = "pkg-test-del"
self.db.session.add(pkg_del)
self.db.session.flush()
config["storage.lobdir"] = tempfile.mkdtemp(prefix="faf")
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg_del.save_lob("package", sample, truncate=True)
self.assertTrue(pkg_del.has_lob("package"))
# add build and package and lob that will not be deleted
build = Build()
build.base_package_name = "build_unassigned"
build.epoch = 0
build.version = "1.2.3"
build.release = "20.fc23"
self.db.session.add(build)
pkg_stay = Package()
pkg_stay.build = build
pkg_stay.arch = self.db.session.query(Arch).first()
pkg_stay.name = "pkg-test-stay"
self.db.session.add(pkg_stay)
self.db.session.flush()
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg_stay.save_lob("package", sample, truncate=True)
self.assertTrue(pkg_stay.has_lob("package"))
init_bosra = self.db.session.query(BuildOpSysReleaseArch).count()
bosra = self.db.session.query(BuildOpSysReleaseArch).count()
# delete release f24 with assigned repos and builds with lobs
self.assertEqual(self.call_action("releasedel", {
"opsys": "fedora",
"opsys-release": "24",
}), 0)
self.assertEqual(bosra, init_bosra)
self.assertTrue(pkg_stay.has_lob("package"))
self.assertFalse(pkg_del.has_lob("package"))
self.assertEqual(self.db.session.query(Report).count(), 2)
self.assertEqual(self.db.session.query(Problem).count(), 2)
# delete release f20 with assigned bugs, reports and problems
self.assertEqual(self.call_action("releasedel", {
"opsys": "fedora",
"opsys-release": "20",
}), 0)
self.assertEqual(self.db.session.query(Report).count(), 0)
self.assertEqual(self.db.session.query(Problem).count(), 0)
# attempt to delete deleted release f20
self.assertEqual(self.call_action("releasedel", {
"opsys": "fedora",
"opsys-release": "20",
}), 1)