def test_store_rpm_deps(self):
"""
"""
# add required stuff to db
arch = Arch()
arch.name = "noarch"
self.db.session.add(arch)
build = Build()
build.base_package_name = "sample"
build.version = "1"
build.release = "1"
build.epoch = "0"
self.db.session.add(build)
pkg = Package()
pkg.name = "sample"
pkg.pkgtype = "rpm"
pkg.arch = arch
pkg.build = build
self.db.session.add(pkg)
self.db.session.flush()
# save sample rpm
sample_rpm = glob.glob("sample_rpms/sample*.rpm")[0]
with open(sample_rpm, mode='rb') as sample:
pkg.save_lob("package", sample, truncate=True)
# get dependencies
res = store_rpm_deps(self.db, pkg, nogpgcheck=True)
self.assertIs(res, True)
expected_deps = [
("PROVIDES", "sample", 8),
("PROVIDES", "/sample", 0),
("PROVIDES", "nothing-new", 0),
(
"REQUIRES",
"nothing",
0,
),
(
"CONFLICTS",
"surprisingly-nothing",
0,
),
]
found_deps = []
for dep in self.db.session.query(PackageDependency).all():
found_deps.append((dep.type, dep.name, dep.flags))
for dep in expected_deps:
self.assertIn(dep, found_deps)
def restore_package_deps(self):
print('Restoring package dependencies from rpms')
for package in self.ses.query(Package):
store_rpm_deps(self.db, package)
self.ses.commit()
def run(self, cmdline, db):
repo_instances = []
for repo in db.session.query(Repo):
if cmdline.NAME and repo.name not in cmdline.NAME:
continue
if cmdline.match_repos not in repo.name:
continue
if not repo.type in repo_types:
self.log_error("No plugin installed to handle repository type "
"{0}, skipping.".format(repo.type))
continue
if repo.opsys_list: # parametrized
self.log_info("Processing parametrized repo '{0}'"
.format(repo.name))
if not repo.arch_list:
self.log_error("Parametrized repository is not assigned"
" with an architecture, skipping")
continue
try:
repo_instances += list(self._get_parametrized_variants(repo))
except: # pylint: disable=bare-except
self.log_error("No valid mirror for repository '{0}', skipping"
.format(repo.name))
continue
elif repo.opsysrelease_list:
self.log_info("Processing repo '{0}' assigned with OpSysRelease"
.format(repo.name))
if not repo.arch_list:
self.log_error("OpSysRelease repository is not assigned"
" with an architecture, skipping")
continue
try:
repo_instances += list(self._get_opsysrelease_variants(repo))
except: # pylint: disable=bare-except
self.log_error("No valid mirror for repository '{0}', skipping"
.format(repo.name))
continue
else:
if any('$' in url.url for url in repo.url_list):
self.log_error("No operating system assigned to"
"parametrized repo '{0}', skipping".format(repo.name))
continue
for arch in repo.arch_list:
try:
repo_instance = {
'instance' : repo_types[repo.type](
repo.name,
[url.url for url in repo.url_list]),
'opsys' : None,
'release' : None,
'arch' : arch.name,
'nogpgcheck': repo.nogpgcheck}
repo_instances.append(repo_instance)
except: # pylint: disable=bare-except
self.log_error("No valid mirror for repository '{0}', skipping"
.format(repo.name))
continue
cmdline.name_prefix = cmdline.name_prefix.lower()
architectures = dict((x.name, x) for x in get_archs(db))
for repo_instance in repo_instances:
self.log_info("Processing repository '{0}' URL: '{1}'"
.format(repo_instance['instance'].name,
repo_instance['instance'].urls))
pkglist = \
repo_instance['instance'].list_packages(list(architectures.keys()))
total = len(pkglist)
self.log_info("Repository has '{0}' packages".format(total))
for num, pkg in enumerate(pkglist):
self.log_debug("[{0} / {1}] Processing package {2}"
.format(num + 1, total, pkg["name"]))
if not pkg["name"].lower().startswith(cmdline.name_prefix):
self.log_debug("Skipped package {0}"
.format(pkg["name"]))
continue
arch = architectures.get(pkg["arch"], None)
if not arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(pkg["arch"]))
continue
repo_arch = architectures.get(repo_instance["arch"], None)
if not repo_arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(repo_instance["arch"]))
continue
build = (db.session.query(Build)
.filter(Build.base_package_name ==
pkg["base_package_name"])
.filter(Build.version == pkg["version"])
.filter(Build.release == pkg["release"])
.filter(Build.epoch == pkg["epoch"])
.first())
if not build:
self.log_debug("Adding build {0}-{1}".format(
pkg["base_package_name"], pkg["version"]))
build = Build()
build.base_package_name = pkg["base_package_name"]
build.version = pkg["version"]
build.release = pkg["release"]
build.epoch = pkg["epoch"]
db.session.add(build)
db.session.flush()
build_arch = (db.session.query(BuildArch)
.filter(BuildArch.build_id == build.id)
.filter(BuildArch.arch_id == arch.id)
.first())
if not build_arch:
build_arch = BuildArch()
build_arch.build = build
build_arch.arch = arch
db.session.add(build_arch)
db.session.flush()
build_opsysrelease_arch = (
db.session.query(BuildOpSysReleaseArch)
.join(Build)
.join(OpSysRelease)
.join(Arch)
.filter(Build.id == build.id)
.filter(OpSys.name == repo_instance['opsys'])
.filter(OpSysRelease.version == repo_instance['release'])
.filter(Arch.name == repo_instance['arch'])
.first())
if not build_opsysrelease_arch and repo_instance['release'] and repo_instance['opsys']:
self.log_info("Adding link between build {0}-{1} "
"operating system '{2}', release '{3} and "
"architecture {4}".format(pkg["base_package_name"],
pkg["version"], repo_instance['opsys'],
repo_instance['release'], repo_instance['arch']))
opsysrelease = (
db.session.query(OpSysRelease)
.filter(OpSys.name == repo_instance['opsys'])
.filter(OpSysRelease.version == repo_instance['release'])
.first())
bosra = BuildOpSysReleaseArch()
bosra.build = build
bosra.opsysrelease = opsysrelease
bosra.arch = repo_arch
db.session.add(bosra)
db.session.flush()
package = (db.session.query(Package)
.filter(Package.name == pkg["name"])
.filter(Package.pkgtype == pkg["type"])
.filter(Package.build == build)
.filter(Package.arch == arch)
.first())
if not package:
self.log_info("Adding package {0}".format(pkg["filename"]))
package = Package()
package.name = pkg["name"]
package.pkgtype = pkg["type"]
package.arch = arch
package.build = build
db.session.add(package)
db.session.flush()
if cmdline.no_download_rpm:
continue
# Catching too general exception Exception
# pylint: disable-msg=W0703
try:
self.log_info("Downloading {0}".format(pkg["url"]))
self._download(package, "package", pkg["url"])
except Exception as exc:
self.log_error("Exception ({0}) after multiple attempts"
" while trying to download {1},"
" skipping.".format(exc, pkg["url"]))
db.session.delete(package)
db.session.flush()
continue
# pylint: enable-msg=W0703
res = True
if pkg["type"] == "rpm":
res = store_rpm_deps(db, package, repo_instance['nogpgcheck'])
if not res:
self.log_error("Post-processing failed, skipping")
db.session.delete(package)
db.session.flush()
continue
if cmdline.no_store_rpm:
try:
package.del_lob("package")
self.log_info("Package deleted.")
except Exception as exc: # pylint: disable=broad-except
self.log_error("Error deleting the RPM file.")
else:
self.log_debug("Known package {0}".format(pkg["filename"]))
def run(self, cmdline, db):
repo_instances = []
for repo in db.session.query(Repo):
if cmdline.NAME and repo.name not in cmdline.NAME:
continue
if not repo.type in repo_types:
self.log_error("No plugin installed to handle repository type "
"{0}, skipping.".format(repo.type))
continue
if repo.opsys_list: # parametrized
self.log_info("Processing parametrized repo '{0}'"
.format(repo.name))
if not repo.arch_list:
self.log_error("Parametrized repository is not assigned"
" with an architecture")
return 1
try:
repo_instances += list(self._get_parametrized_variants(repo))
except:
self.log_error("No valid mirror for repository {0}"
.format(repo.name))
return 1
elif repo.opsysrelease_list:
self.log_info("Processing repo '{0}' assigned with OpSysRelease"
.format(repo.name))
if not repo.arch_list:
self.log_error("OpSysRelease repository is not assigned"
" with an architecture")
return 1
try:
repo_instances += list(self._get_opsysrelease_variants(repo))
except:
self.log_error("No valid mirror for repository {0}"
.format(repo.name))
return 1
else:
if any('$' in url.url for url in repo.url_list):
self.log_error("No operating system assigned to"
"parametrized repo '{0}".format(repo.name))
return 1
for arch in repo.arch_list:
try:
repo_instance = {
'instance' : repo_types[repo.type](repo.name,
[url.url for url in repo.url_list]),
'opsys' : None,
'release' : None,
'arch' : arch.name}
repo_instances.append(repo_instance)
except:
self.log_error("No valid mirror for repository {0}"
.format(repo.name))
return 1
cmdline.name_prefix = cmdline.name_prefix.lower()
architectures = dict((x.name, x) for x in get_archs(db))
for repo_instance in repo_instances:
self.log_info("Processing repository '{0}' URL: '{1}'"
.format(repo_instance['instance'].name,
repo_instance['instance'].urls))
pkglist = \
repo_instance['instance'].list_packages(architectures.keys())
total = len(pkglist)
self.log_info("Repository has '{0}' packages".format(total))
for num, pkg in enumerate(pkglist):
self.log_debug("[{0} / {1}] Processing package {2}"
.format(num + 1, total, pkg["name"]))
if not pkg["name"].lower().startswith(cmdline.name_prefix):
self.log_debug("Skipped package {0}"
.format(pkg["name"]))
continue
arch = architectures.get(pkg["arch"], None)
if not arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(pkg["arch"]))
continue
repo_arch = architectures.get(repo_instance["arch"], None)
if not repo_arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(repo_instance["arch"]))
continue
build = (db.session.query(Build)
.filter(Build.base_package_name ==
pkg["base_package_name"])
.filter(Build.version == pkg["version"])
.filter(Build.release == pkg["release"])
.filter(Build.epoch == pkg["epoch"])
.first())
if not build:
self.log_debug("Adding build {0}-{1}".format(
pkg["base_package_name"], pkg["version"]))
build = Build()
build.base_package_name = pkg["base_package_name"]
build.version = pkg["version"]
build.release = pkg["release"]
build.epoch = pkg["epoch"]
db.session.add(build)
db.session.flush()
build_arch = (db.session.query(BuildArch)
.filter(BuildArch.build_id == build.id)
.filter(BuildArch.arch_id == arch.id)
.first())
if not build_arch:
build_arch = BuildArch()
build_arch.build = build
build_arch.arch = arch
db.session.add(build_arch)
db.session.flush()
build_opsysrelease_arch = (
db.session.query(BuildOpSysReleaseArch)
.join(Build)
.join(OpSysRelease)
.join(Arch)
.filter(Build.id == build.id)
.filter(OpSys.name == repo_instance['opsys'])
.filter(OpSysRelease.version == repo_instance['release'])
.filter(Arch.name == repo_instance['arch'])
.first())
if not build_opsysrelease_arch and repo_instance['release'] and repo_instance['opsys']:
self.log_info("Adding link between build {0}-{1} "
"operating system '{2}', release '{3} and "
"architecture {4}".format(pkg["base_package_name"],
pkg["version"], repo_instance['opsys'],
repo_instance['release'], repo_instance['arch']))
opsysrelease = (
db.session.query(OpSysRelease)
.filter(OpSys.name == repo_instance['opsys'])
.filter(OpSysRelease.version == repo_instance['release'])
.first())
bosra = BuildOpSysReleaseArch()
bosra.build = build
bosra.opsysrelease = opsysrelease
bosra.arch = repo_arch
db.session.add(bosra)
db.session.flush()
package = (db.session.query(Package)
.filter(Package.name == pkg["name"])
.filter(Package.pkgtype == pkg["type"])
.filter(Package.build == build)
.filter(Package.arch == arch)
.first())
if not package:
self.log_info("Adding package {0}".format(pkg["filename"]))
package = Package()
package.name = pkg["name"]
package.pkgtype = pkg["type"]
package.arch = arch
package.build = build
db.session.add(package)
db.session.flush()
if cmdline.no_download_rpm:
continue
# Catching too general exception Exception
# pylint: disable-msg=W0703
try:
self.log_info("Downloading {0}".format(pkg["url"]))
self._download(package, "package", pkg["url"])
except Exception as exc:
self.log_error("Exception ({0}) after multiple attemps"
" while trying to download {1},"
" skipping.".format(exc, pkg["url"]))
db.session.delete(package)
db.session.flush()
continue
# pylint: enable-msg=W0703
res = True
if pkg["type"] == "rpm":
res = store_rpm_deps(db, package, repo.nogpgcheck)
if not res:
self.log_error("Post-processing failed, skipping")
db.session.delete(package)
db.session.flush()
continue
if cmdline.no_store_rpm:
try:
package.del_lob("package")
self.log_info("Package deleted.")
except Exception as exc:
self.log_error("Error deleting the RPM file.")
else:
self.log_debug("Known package {0}".format(pkg["filename"]))
def run(self, cmdline, db):
repo_instances = []
for repo in db.session.query(Repo):
if cmdline.NAME and repo.name not in cmdline.NAME:
continue
if not repo.type in repo_types:
self.log_error("No plugin installed to handle repository type "
"{0}, skipping.".format(repo.type))
continue
if "$" in repo.url: # parametrized
self.log_info("Processing parametrized repo '{0}'"
.format(repo.name))
if not repo.opsys_list:
self.log_error("Parametrized repository is not assigned"
" with an operating system")
return 1
if not repo.arch_list:
self.log_error("Parametrized repository is not assigned"
" with an architecture")
return 1
repo_instances += list(self._get_parametrized_variants(repo))
else:
repo_instance = repo_types[repo.type](repo.name, repo.url)
repo_instances.append(repo_instance)
cmdline.name_prefix = cmdline.name_prefix.lower()
architectures = dict((x.name, x) for x in get_archs(db))
for repo_instance in repo_instances:
self.log_info("Processing repository '{0}' URL: '{1}'"
.format(repo_instance.name, repo_instance.urls[0]))
pkglist = repo_instance.list_packages(architectures.keys())
total = len(pkglist)
self.log_info("Repository has '{0}' packages".format(total))
for num, pkg in enumerate(pkglist):
self.log_debug("[{0} / {1}] Processing package {2}"
.format(num + 1, total, pkg["name"]))
if not pkg["name"].lower().startswith(cmdline.name_prefix):
self.log_debug("Skipped package {0}"
.format(pkg["name"]))
continue
arch = architectures.get(pkg["arch"], None)
if not arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(pkg["arch"]))
continue
build = (db.session.query(Build)
.filter(Build.base_package_name ==
pkg["base_package_name"])
.filter(Build.version == pkg["version"])
.filter(Build.release == pkg["release"])
.filter(Build.epoch == pkg["epoch"])
.first())
if not build:
self.log_debug("Adding build {0}-{1}".format(
pkg["base_package_name"], pkg["version"]))
build = Build()
build.base_package_name = pkg["base_package_name"]
build.version = pkg["version"]
build.release = pkg["release"]
build.epoch = pkg["epoch"]
db.session.add(build)
build_arch = BuildArch()
build_arch.build = build
build_arch.arch = arch
db.session.add(build_arch)
db.session.flush()
package = (db.session.query(Package)
.filter(Package.name == pkg["name"])
.filter(Package.pkgtype == pkg["type"])
.filter(Package.build == build)
.filter(Package.arch == arch)
.first())
if not package:
self.log_info("Adding package {0}".format(pkg["filename"]))
package = Package()
package.name = pkg["name"]
package.pkgtype = pkg["type"]
package.arch = arch
package.build = build
db.session.add(package)
db.session.flush()
if cmdline.no_download_rpm:
continue
# Catching too general exception Exception
# pylint: disable-msg=W0703
try:
self.log_info("Downloading {0}".format(pkg["url"]))
self._download(package, "package", pkg["url"])
except Exception as exc:
self.log_error("Exception ({0}) after multiple attemps"
" while trying to download {1},"
" skipping.".format(exc, pkg["url"]))
db.session.delete(package)
db.session.flush()
continue
# pylint: enable-msg=W0703
res = True
if pkg["type"] == "rpm":
res = store_rpm_deps(db, package, repo.nogpgcheck)
if not res:
self.log_error("Post-processing failed, skipping")
db.session.delete(package)
db.session.flush()
continue
if cmdline.no_store_rpm:
try:
package.del_lob("package")
self.log_info("Package deleted.")
except Exception as exc:
self.log_error("Error deleting the RPM file.")
else:
self.log_debug("Known package {0}".format(pkg["filename"]))
def restore_package_deps(self):
print "Restoring package dependencies from rpms"
for package in self.ses.query(Package):
store_rpm_deps(self.db, package)
self.ses.commit()
def run(self, cmdline, db):
repo_instances = []
for repo in db.session.query(Repo):
if cmdline.NAME and repo.name not in cmdline.NAME:
continue
if not repo.type in repo_types:
self.log_error("No plugin installed to handle repository type "
"{0}, skipping.".format(repo.type))
continue
if "$" in repo.url: # parametrized
self.log_info("Processing parametrized repo '{0}'"
.format(repo.name))
if not repo.opsys_list:
self.log_error("Parametrized repository is not assigned"
" with an operating system")
return 1
if not repo.arch_list:
self.log_error("Parametrized repository is not assigned"
" with an architecture")
return 1
repo_instances += list(self._get_parametrized_variants(repo))
else:
repo_instance = repo_types[repo.type](repo.name, repo.url)
repo_instances.append(repo_instance)
architectures = map(lambda x: x.name, db.session.query(Arch))
for repo_instance in repo_instances:
self.log_info("Processing repository '{0}' URL: '{1}'"
.format(repo_instance.name, repo_instance.urls[0]))
pkglist = repo_instance.list_packages(architectures)
total = len(pkglist)
self.log_info("Repository has '{0}' packages".format(total))
for num, pkg in enumerate(pkglist):
self.log_debug("[{0} / {1}] Processing package {2}"
.format(num + 1, total, pkg["name"]))
arch = get_arch_by_name(db, pkg["arch"])
if not arch:
self.log_error("Architecture '{0}' not found, skipping"
.format(pkg["arch"]))
continue
build = (db.session.query(Build)
.filter(Build.base_package_name ==
pkg["base_package_name"])
.filter(Build.version == pkg["version"])
.filter(Build.release == pkg["release"])
.filter(Build.epoch == pkg["epoch"])
.first())
if not build:
self.log_debug("Adding build {0}-{1}".format(
pkg["base_package_name"], pkg["version"]))
build = Build()
build.base_package_name = pkg["base_package_name"]
build.version = pkg["version"]
build.release = pkg["release"]
build.epoch = pkg["epoch"]
db.session.add(build)
build_arch = BuildArch()
build_arch.build = build
build_arch.arch = arch
db.session.add(build_arch)
db.session.flush()
package = (db.session.query(Package)
.filter(Package.name == pkg["name"])
.filter(Package.pkgtype == pkg["type"])
.filter(Package.build == build)
.filter(Package.arch == arch)
.first())
if not package:
self.log_info("Adding package {0}".format(pkg["filename"]))
package = Package()
package.name = pkg["name"]
package.pkgtype = pkg["type"]
package.arch = arch
package.build = build
db.session.add(package)
db.session.flush()
if cmdline.no_download_rpm:
continue
# Catching too general exception Exception
# pylint: disable-msg=W0703
try:
self.log_info("Downloading {0}".format(pkg["url"]))
self._download(package, "package", pkg["url"])
except Exception as exc:
self.log_error("Exception ({0}) after multiple attemps"
" while trying to download {1},"
" skipping.".format(exc, pkg["url"]))
db.session.delete(package)
db.session.flush()
continue
# pylint: enable-msg=W0703
res = True
if pkg["type"] == "rpm":
res = store_rpm_deps(db, package, repo.nogpgcheck)
if not res:
self.log_error("Post-processing failed, skipping")
db.session.delete(package)
db.session.flush()
continue
if cmdline.no_store_rpm:
try:
package.del_lob("package")
self.log_info("Package deleted.")
except Exception as exc:
self.log_error("Error deleting the RPM file.")
else:
self.log_debug("Known package {0}".format(pkg["filename"]))