def _diff_chlog(oldlogs, newlogs):
newlog = _fmt_chlog(newlogs)
oldlog = _fmt_chlog(oldlogs)
chlog = []
skip = False
for line in newlog:
if line.startswith("*"):
if not line in oldlog:
chlog.append(line)
skip = False
else:
skip = True
continue
else:
if not skip:
chlog.append(line)
return chlog
def _diff_chlog(oldlogs, newlogs):
newlog = _fmt_chlog(newlogs)
oldlog = _fmt_chlog(oldlogs)
chlog = []
skip = False
for line in newlog:
if line.startswith("*"):
if not line in oldlog:
chlog.append(line)
skip = False
else:
skip = True
continue
else:
if not skip:
chlog.append(line)
return chlog
def packages(self):
cachekey = "%s%s" % ("repopackages", self.id)
pkgs = cache.get(cachekey)
if not pkgs is None:
return pkgs
pkgs = {}
ARCHS = [ str(x) for x in Arch.objects.all() ]
PLATS = set( repo.platform.name for repo in self.comps )
PLATS.add(self.platform.name)
repo_pkg_meta = self.pkg_meta
#sackpkgs = self.yumsack.returnPackages()
#sackpkgs.sort(cmp=lambda x,y: x.verCMP(y), reverse=True)
for pkg in self.yumsack.returnPackages():
if not pkg.arch in ARCHS:
continue
# have we seen a binary from the same base package before ?
if pkg.base_package_name in pkgs:
# yes, check where it lives
if pkg.repoid in pkgs[pkg.base_package_name]:
# same repo
if pkg.ver == pkgs[pkg.base_package_name][pkg.repoid]["version"]:
# same version, append to binaries list
pkgs[pkg.base_package_name][pkg.repoid]['binaries'].add(pkg)
pkgs[pkg.base_package_name][pkg.repoid]['binary_names'].add(pkg.name)
if not pkg.name.endswith("debuginfo") and not pkg.name.endswith("debugsource")and not pkg.name.endswith("-doc") and not pkg.name.endswith("-tests") and not pkg.name.endswith("-devel"):
pkgs[pkg.base_package_name][pkg.repoid].update({"description" : pkg.description, "summary" : pkg.summary})
else:
# oh no different version!
pkgs[pkg.base_package_name][pkg.repoid]["messages"].append("Warning: %s exists in the same repo with different version %s!" % (pkg.name, pkg.ver))
# no need to look further
continue
else:
pkgs[pkg.base_package_name] = {}
pkg_meta = _get_pkg_meta(pkg.base_package_name, PLATS, repo_pkg_meta)
pkgs[pkg.base_package_name][pkg.repoid] = { "version" : pkg.ver,
"release" : pkg.rel,
"changelog" : _fmt_chlog(pkg.changelog),
"license" : pkg.license,
"binaries" : set([pkg]),
"binary_names": set([pkg.name]),
"meta" : pkg_meta,
"messages" : [],
}
if not pkg.name.endswith("debuginfo") and not pkg.name.endswith("debugsource")and not pkg.name.endswith("-doc") and not pkg.name.endswith("-tests") and not pkg.name.endswith("-devel"):
pkgs[pkg.base_package_name][pkg.repoid].update({"description" : pkg.description, "summary" : pkg.summary})
cachelife = (60 * 5) if self.is_live else (60 * 60 * 24)
cache.set(cachekey, pkgs, cachelife)
return pkgs
def packages(self):
cachekey = "%s%s" % ("repopackages", self.id)
pkgs = cache.get(cachekey)
if not pkgs is None:
return pkgs
pkgs = {}
ARCHS = [str(x) for x in Arch.objects.all()]
PLATS = set(repo.platform.name for repo in self.comps)
PLATS.add(self.platform.name)
repo_pkg_meta = self.pkg_meta
#sackpkgs = self.yumsack.returnPackages()
#sackpkgs.sort(cmp=lambda x,y: x.verCMP(y), reverse=True)
for pkg in self.yumsack.returnPackages():
if not pkg.arch in ARCHS:
continue
# have we seen a binary from the same base package before ?
if pkg.base_package_name in pkgs:
# yes, check where it lives
if pkg.repoid in pkgs[pkg.base_package_name]:
# same repo
if pkg.ver == pkgs[pkg.base_package_name][
pkg.repoid]["version"]:
# same version, append to binaries list
pkgs[pkg.base_package_name][
pkg.repoid]['binaries'].add(pkg)
pkgs[pkg.base_package_name][
pkg.repoid]['binary_names'].add(pkg.name)
if not pkg.name.endswith(
"debuginfo") and not pkg.name.endswith(
"debugsource") and not pkg.name.endswith(
"-doc") and not pkg.name.endswith(
"-tests"
) and not pkg.name.endswith("-devel"):
pkgs[pkg.base_package_name][pkg.repoid].update({
"description":
pkg.description,
"summary":
pkg.summary
})
else:
# oh no different version!
pkgs[pkg.base_package_name][
pkg.repoid]["messages"].append(
"Warning: %s exists in the same repo with different version %s!"
% (pkg.name, pkg.ver))
# no need to look further
continue
else:
pkgs[pkg.base_package_name] = {}
pkg_meta = _get_pkg_meta(pkg.base_package_name, PLATS,
repo_pkg_meta)
pkgs[pkg.base_package_name][pkg.repoid] = {
"version": pkg.ver,
"release": pkg.rel,
"changelog": _fmt_chlog(pkg.changelog),
"license": pkg.license,
"binaries": set([pkg]),
"binary_names": set([pkg.name]),
"meta": pkg_meta,
"messages": [],
}
if not pkg.name.endswith("debuginfo") and not pkg.name.endswith(
"debugsource") and not pkg.name.endswith(
"-doc") and not pkg.name.endswith(
"-tests") and not pkg.name.endswith("-devel"):
pkgs[pkg.base_package_name][pkg.repoid].update({
"description":
pkg.description,
"summary":
pkg.summary
})
cachelife = (60 * 5) if self.is_live else (60 * 60 * 24)
cache.set(cachekey, pkgs, cachelife)
return pkgs
def _diff_sacks(newrepo, oldrepo, progress_cb):
is_livediff = newrepo.is_live and oldrepo.is_live
newprjsack = newrepo.prjsack
oldprjsack = oldrepo.prjsack
repo_pkg_meta = newrepo.pkg_meta
platforms = set( repo.platform.name for repo in newrepo.comps )
platforms.add(newrepo.platform.name)
# key_tuple = ( repo id, repo string, pkg base name )
# { key_tuple : { binaries : set[] , chlog : [] } }
added = defaultdict(dict)
# { key_tuple : set[] }
removed = defaultdict(set)
# { key_tuple : { ovr : old version , oa : old arch , nvr : new version , na : new arch , chlo : changelog diff } }
modified = {}
obsoleted = {}
obsoletes = {}
ARCHS = set([ str(x) for x in Arch.objects.all() ])
ARCHS.add("src")
#toskip = set([ "debuginfo", "debugsource", "devel", "tests" ])
toskip = set([ "debuginfo", "debugsource" ])
# first go through the new repo collection
repoids = {}
if is_livediff and newrepo.comps:
for repo in newrepo.comps:
repoids[repo.yumrepoid] = repo
else:
repoids[newrepo.yumrepoid] = newrepo
repo_count = len(repoids.keys()) * 2
done_count = 0
for repoid, repo in repoids.items():
done_count += 1
progress_cb(math.ceil((float(done_count)/repo_count)*100))
repo_str = str(repo)
prjsack = list(repo.prjsack)
project = None
if repo.is_live and len(prjsack) == 1:
project = prjsack[0].request_target
if not project or not project in oldprjsack:
try:
project = prjsack[0].request_source.get()
except ObjectDoesNotExist:
project = None
if project and project in oldprjsack:
project = str(project.name)
else:
project = None
old_comparable = _find_comparable_component(repo, oldrepo, project=project)
if old_comparable:
old_comparable = old_comparable.yumsack
else:
old_comparable = oldrepo.yumsack
pkgs = []
if is_livediff:
pkgs = repo.yumsack.returnPackages()
else:
pkgs = repo.yumsack.returnNewestByName()
for pkg in pkgs:
# skip archs we don't care about
if not pkg.arch in ARCHS:
continue
# skip debuginfo and debugsource rpms
suffix = pkg.name.split("-")[-1]
if suffix in toskip:
continue
key_tuple = ( repo.id, repo_str, pkg.base_package_name )
nvr = "%s - %s-%s" % ( pkg.name, pkg.ver, pkg.rel)
oldpkgs = list(old_comparable.searchNames(pkg.name))
oldpkgs.sort(cmp=lambda x,y: x.verCMP(y), reverse=True)
oldpkgs = oldpkgs[:1]
for oldpkg in oldpkgs:
#if not oldpkg.arch in ARCHS:
# continue
#if not pkg.arch == oldpkg.arch and ( not pkg.arch == "noarch" or not oldpkg.arch == "noarch" ):
# continue
if not key_tuple in modified:
# check for a change in version or release number
if not pkg.ver == oldpkg.ver or not pkg.rel == oldpkg.rel:
# get new changelog entries
chlog = _diff_chlog(oldpkg.changelog, pkg.changelog)
# a version change is reported even without changelog
# while a release change is ignored in that case
if len(chlog) or not pkg.ver == oldpkg.ver:
if not len(chlog):
chlog.append("No new changelog entries!")
modified.update({ key_tuple : { "sense" : "Updated" if pkg.verCMP(oldpkg) == 1 else "Reverted",
"ovr" : "%s-%s" % (oldpkg.ver, oldpkg.rel),
"oa" : oldpkg.arch,
"nvr" : "%s-%s" % (pkg.ver, pkg.rel),
"na" : pkg.arch,
"ba" : set(),
"br" : set(),
"chlo" : chlog,
"meta" : _get_pkg_meta(pkg.base_package_name, platforms, repo_pkg_meta),
"unmet_reqs" : set()
}})
#if key_tuple in modified:
# modified[key_tuple]["unmet_reqs"].update(_find_unmet_reqs(pkg, newsack, oldsack = oldsack))
# we don't need to look further
break
if not len(oldpkgs):
if not key_tuple in added:
added[key_tuple]["binaries"] = set()
added[key_tuple]["chlog"] = _fmt_chlog(pkg.changelog)
added[key_tuple]["meta"] = _get_pkg_meta(pkg.base_package_name, platforms, repo_pkg_meta)
added[key_tuple]["unmet_reqs"] = set()
added[key_tuple]["binaries"].add(nvr)
#added[key_tuple]["unmet_reqs"].update(_find_unmet_reqs(pkg, newsack, oldsack = oldsack))
if pkg.obsoletes:
obsoletes[nvr] = pkg.obsoletes
oldpkgs_by_repoid = defaultdict(list)
# now go through the old repo collection
repoids = {}
if is_livediff and oldrepo.comps:
for repo in oldrepo.comps:
repoids[repo.yumrepoid] = repo
else:
repoids[oldrepo.yumrepoid] = oldrepo
repo_count = len(repoids.keys())
done_count = 0
for repoid, repo in repoids.items():
done_count += 1
progress_cb(math.ceil((((float(done_count)/repo_count)*100)/2)+50))
pkgs = repo.yumsack.returnPackages()
repo_str = str(repo)
# cache the specific comparable component in the older repo
prjsack = list(repo.prjsack)
project = None
if repo.is_live and len(prjsack) == 1:
try:
project = prjsack[0].request_source.get()
except ObjectDoesNotExist:
project = prjsack[0].request_target
if project and project in newprjsack:
project = str(project.name)
else:
project = None
new_comparable = _find_comparable_component(repo, newrepo, project=project)
if new_comparable:
new_comparable_id = new_comparable.id
new_comparable_str = str(new_comparable)
new_comparable = new_comparable.yumsack
else:
new_comparable_id = newrepo.id
new_comparable_str = str(newrepo)
new_comparable = newrepo.yumsack
for pkg in pkgs:
if not pkg.arch in ARCHS:
continue
# skip debuginfo and debugsource rpms
suffix = pkg.name.split("-")[-1]
if suffix in toskip:
continue
# look for removed packages
if not list(new_comparable.searchNames(pkg.name)):
key_tuple = ( new_comparable_id, new_comparable_str, pkg.base_package_name)
if key_tuple in modified:
modified[key_tuple]['br'].add(pkg.name)
else:
oldpkgs_by_repoid[pkg.repoid].append(pkg)
for repoid, pkgs in oldpkgs_by_repoid.items():
repo = repoids.get(repoid, oldrepo)
repo_str = str(repo)
for pkg in pkgs:
nvr = "%s - %s-%s" % ( pkg.name, pkg.ver, pkg.rel)
removed[( repo.id, repo_str, pkg.base_package_name )].add(nvr)
for obsby, obss in obsoletes.items():
for obs in obss:
if pkg.inPrcoRange('provides', obs):
obsoleted.update({ nvr : obsby })
# merge added binaries to corresponding entries in modified
# leftover packages are really new or have been completely removed
toremove = set()
for key_tuple in modified:
repoid, repostr, pkg = key_tuple
if key_tuple in added:
modified[key_tuple]["ba"] = added[key_tuple]['binaries']
del(added[key_tuple])
diff = { "added" : dict(added),
"removed" : dict(removed),
"modified" : modified,
"obsoleted" : obsoleted}
progress_cb( 100 )
return diff
def _diff_sacks(newrepo, oldrepo, progress_cb):
is_livediff = newrepo.is_live and oldrepo.is_live
newprjsack = newrepo.prjsack
oldprjsack = oldrepo.prjsack
repo_pkg_meta = newrepo.pkg_meta
platforms = set(repo.platform.name for repo in newrepo.comps)
platforms.add(newrepo.platform.name)
# key_tuple = ( repo id, repo string, pkg base name )
# { key_tuple : { binaries : set[] , chlog : [] } }
added = defaultdict(dict)
# { key_tuple : set[] }
removed = defaultdict(set)
# { key_tuple : { ovr : old version , oa : old arch , nvr : new version , na : new arch , chlo : changelog diff } }
modified = {}
obsoleted = {}
obsoletes = {}
ARCHS = set([str(x) for x in Arch.objects.all()])
ARCHS.add("src")
#toskip = set([ "debuginfo", "debugsource", "devel", "tests" ])
toskip = set(["debuginfo", "debugsource"])
# first go through the new repo collection
repoids = {}
if is_livediff and newrepo.comps:
for repo in newrepo.comps:
repoids[repo.yumrepoid] = repo
else:
repoids[newrepo.yumrepoid] = newrepo
repo_count = len(repoids.keys()) * 2
done_count = 0
for repoid, repo in repoids.items():
done_count += 1
progress_cb(math.ceil((float(done_count) / repo_count) * 100))
repo_str = str(repo)
prjsack = list(repo.prjsack)
project = None
if repo.is_live and len(prjsack) == 1:
project = prjsack[0].request_target
if not project or not project in oldprjsack:
try:
project = prjsack[0].request_source.get()
except ObjectDoesNotExist:
project = None
if project and project in oldprjsack:
project = str(project.name)
else:
project = None
old_comparable = _find_comparable_component(repo,
oldrepo,
project=project)
if old_comparable:
old_comparable = old_comparable.yumsack
else:
old_comparable = oldrepo.yumsack
pkgs = []
if is_livediff:
pkgs = repo.yumsack.returnPackages()
else:
pkgs = repo.yumsack.returnNewestByName()
for pkg in pkgs:
# skip archs we don't care about
if not pkg.arch in ARCHS:
continue
# skip debuginfo and debugsource rpms
suffix = pkg.name.split("-")[-1]
if suffix in toskip:
continue
key_tuple = (repo.id, repo_str, pkg.base_package_name)
nvr = "%s - %s-%s" % (pkg.name, pkg.ver, pkg.rel)
oldpkgs = list(old_comparable.searchNames(pkg.name))
oldpkgs.sort(cmp=lambda x, y: x.verCMP(y), reverse=True)
oldpkgs = oldpkgs[:1]
for oldpkg in oldpkgs:
#if not oldpkg.arch in ARCHS:
# continue
#if not pkg.arch == oldpkg.arch and ( not pkg.arch == "noarch" or not oldpkg.arch == "noarch" ):
# continue
if not key_tuple in modified:
# check for a change in version or release number
if not pkg.ver == oldpkg.ver or not pkg.rel == oldpkg.rel:
# get new changelog entries
chlog = _diff_chlog(oldpkg.changelog, pkg.changelog)
# a version change is reported even without changelog
# while a release change is ignored in that case
if len(chlog) or not pkg.ver == oldpkg.ver:
if not len(chlog):
chlog.append("No new changelog entries!")
modified.update({
key_tuple: {
"sense":
"Updated"
if pkg.verCMP(oldpkg) == 1 else "Reverted",
"ovr":
"%s-%s" % (oldpkg.ver, oldpkg.rel),
"oa":
oldpkg.arch,
"nvr":
"%s-%s" % (pkg.ver, pkg.rel),
"na":
pkg.arch,
"ba":
set(),
"br":
set(),
"chlo":
chlog,
"meta":
_get_pkg_meta(pkg.base_package_name,
platforms, repo_pkg_meta),
"unmet_reqs":
set()
}
})
#if key_tuple in modified:
# modified[key_tuple]["unmet_reqs"].update(_find_unmet_reqs(pkg, newsack, oldsack = oldsack))
# we don't need to look further
break
if not len(oldpkgs):
if not key_tuple in added:
added[key_tuple]["binaries"] = set()
added[key_tuple]["chlog"] = _fmt_chlog(pkg.changelog)
added[key_tuple]["meta"] = _get_pkg_meta(
pkg.base_package_name, platforms, repo_pkg_meta)
added[key_tuple]["unmet_reqs"] = set()
added[key_tuple]["binaries"].add(nvr)
#added[key_tuple]["unmet_reqs"].update(_find_unmet_reqs(pkg, newsack, oldsack = oldsack))
if pkg.obsoletes:
obsoletes[nvr] = pkg.obsoletes
oldpkgs_by_repoid = defaultdict(list)
# now go through the old repo collection
repoids = {}
if is_livediff and oldrepo.comps:
for repo in oldrepo.comps:
repoids[repo.yumrepoid] = repo
else:
repoids[oldrepo.yumrepoid] = oldrepo
repo_count = len(repoids.keys())
done_count = 0
for repoid, repo in repoids.items():
done_count += 1
progress_cb(
math.ceil((((float(done_count) / repo_count) * 100) / 2) + 50))
pkgs = repo.yumsack.returnPackages()
repo_str = str(repo)
# cache the specific comparable component in the older repo
prjsack = list(repo.prjsack)
project = None
if repo.is_live and len(prjsack) == 1:
try:
project = prjsack[0].request_source.get()
except ObjectDoesNotExist:
project = prjsack[0].request_target
if project and project in newprjsack:
project = str(project.name)
else:
project = None
new_comparable = _find_comparable_component(repo,
newrepo,
project=project)
if new_comparable:
new_comparable_id = new_comparable.id
new_comparable_str = str(new_comparable)
new_comparable = new_comparable.yumsack
else:
new_comparable_id = newrepo.id
new_comparable_str = str(newrepo)
new_comparable = newrepo.yumsack
for pkg in pkgs:
if not pkg.arch in ARCHS:
continue
# skip debuginfo and debugsource rpms
suffix = pkg.name.split("-")[-1]
if suffix in toskip:
continue
# look for removed packages
if not list(new_comparable.searchNames(pkg.name)):
key_tuple = (new_comparable_id, new_comparable_str,
pkg.base_package_name)
if key_tuple in modified:
modified[key_tuple]['br'].add(pkg.name)
else:
oldpkgs_by_repoid[pkg.repoid].append(pkg)
for repoid, pkgs in oldpkgs_by_repoid.items():
repo = repoids.get(repoid, oldrepo)
repo_str = str(repo)
for pkg in pkgs:
nvr = "%s - %s-%s" % (pkg.name, pkg.ver, pkg.rel)
removed[(repo.id, repo_str, pkg.base_package_name)].add(nvr)
for obsby, obss in obsoletes.items():
for obs in obss:
if pkg.inPrcoRange('provides', obs):
obsoleted.update({nvr: obsby})
# merge added binaries to corresponding entries in modified
# leftover packages are really new or have been completely removed
toremove = set()
for key_tuple in modified:
repoid, repostr, pkg = key_tuple
if key_tuple in added:
modified[key_tuple]["ba"] = added[key_tuple]['binaries']
del (added[key_tuple])
diff = {
"added": dict(added),
"removed": dict(removed),
"modified": modified,
"obsoleted": obsoleted
}
progress_cb(100)
return diff