def test_metgoal(self):
"""Test that metgoal() works properly."""
pi = progress.GoalTrackerItem("testitem")
self.assertEqual(pi.metgoal(), True)
pi.goalitems = 1
self.assertEqual(pi.metgoal(), False)
pi.items += 1
self.assertEqual(pi.metgoal(), True)
pi.done()
# should work after done()
self.assertEqual(pi.metgoal(), True)
def test_item_before_goal(self):
"""Items must not be able to be set before goal is set."""
def doit():
pi.items = 3
pi = progress.GoalTrackerItem("testitem")
# check can't set items before goal is set
self.assertRaises(RuntimeError, doit)
pi.goalitems = 100
pi.items = 10
pi.reset()
self.assertRaises(RuntimeError, doit)
def test_pairplus1(self):
"""Test that pairplus1() gives proper results."""
pi = progress.GoalTrackerItem("testitem")
# special case before goal is set
self.assertEqual(pi.pairplus1(), "1/1")
pi.goalitems = 100
self.assertEqual(pi.pairplus1(), " 1/100")
pi.items = 100
self.assertEqual(pi.pairplus1(), "100/100")
pi.done()
# should work after done()
self.assertEqual(pi.pairplus1(), "100/100")
def test_elapsed(self):
"""Test GoalTrackerItem elapsed() functionality."""
pi = progress.GoalTrackerItem("testitem")
# special case before goal is set
self.assertTrue(pi.elapsed() == 0.0)
pi.goalitems = 100
self.assertTrue(pi.elapsed() == 0.0)
pi.items = 100
time.sleep(0.20)
# should work before done()
self.assertTrue(pi.elapsed() >= 0.10)
pi.done()
# should work after done()
self.assertTrue(pi.elapsed() >= 0.10)
def test_pctdone(self):
"""Test that pctdone() returns correct values."""
pi = progress.GoalTrackerItem("testitem")
# special case before goal is set
self.assertEqual(pi.pctdone(), 0)
pi.goalitems = 100
self.assertEqual(pi.pctdone(), 0)
pi.items = 50
self.assertEqual(int(pi.pctdone()), 50)
pi.items = 100
self.assertEqual(int(pi.pctdone()), 100)
pi.done()
# should work after done()
self.assertEqual(int(pi.pctdone()), 100)
def do_reversion(pub, ref_pub, target_repo, ref_xport, changes, ignores,
cmp_policy, ref_repo, ref, ref_xport_cfg):
"""Do the repo reversion.
Return 'True' if repo got modified, 'False' otherwise."""
global temp_root, tracker, dry_run, repo_finished, repo_modified
target_cat = target_repo.get_catalog(pub=pub)
ref_cat = fetch_catalog(ref_pub, ref_xport, temp_root)
latest_pkgs = get_latest(target_cat)
latest_ref_pkgs = get_latest(ref_cat)
no_revs = get_matching_pkgs(target_cat, changes)
# We use bulk prefetching for faster transport of the manifests.
# Prefetch requires an intent which it sends to the server. Here
# we just use operation=reversion for all FMRIs.
intent = "operation=reversion;"
# use list() to force the zip() to evaluate
ref_pkgs = list(zip(latest_ref_pkgs.values(), repeat(intent)))
# Retrieve reference manifests.
# Try prefetching manifests in bulk first for faster, parallel
# transport. Retryable errors during prefetch are ignored and
# manifests are retrieved again during the "Reading" phase.
ref_xport.prefetch_manifests(ref_pkgs, progtrack=tracker)
# Need to change the output of mfst_fetch since otherwise we
# would see "Download Manifests x/y" twice, once from the
# prefetch and once from the actual manifest analysis.
tracker.mfst_fetch = progress.GoalTrackerItem(_("Analyzing Manifests"))
tracker.manifest_fetch_start(len(latest_pkgs))
reversioned_pkgs = set()
depend_changes = {}
dups = 0 # target pkg has equal version to ref pkg
new_p = 0 # target pkg not in ref
sucs = 0 # ref pkg is successor to pkg in targ
nrevs = 0 # pkgs requested to not be reversioned by user
for p in latest_pkgs:
# First check if the package is in the list of FMRIs the user
# doesn't want to reversion.
if p in no_revs:
nrevs += 1
tracker.manifest_fetch_progress(completion=True)
continue
# Check if the package is in the ref repo, if not: ignore.
if not p in latest_ref_pkgs:
new_p += 1
tracker.manifest_fetch_progress(completion=True)
continue
pfmri = latest_pkgs[p]
# Ignore if latest package is the same in targ and ref.
if pfmri == latest_ref_pkgs[p]:
dups += 1
tracker.manifest_fetch_progress(completion=True)
continue
# Ignore packages where ref version is higher.
if latest_ref_pkgs[p].is_successor(pfmri):
sucs += 1
tracker.manifest_fetch_progress(completion=True)
continue
# Pull the manifests for target and ref repo.
dm = get_manifest(target_repo, pub, pfmri)
rm = ref_xport.get_manifest(latest_ref_pkgs[p])
tracker.manifest_fetch_progress(completion=True)
tdeps = set()
rdeps = set()
# Diff target and ref manifest.
# action only in targ, action only in ref, common action
ta, ra, ca = manifest.Manifest.comm([dm, rm], cmp_policy=cmp_policy)
# Check for manifest changes.
if not all(use_ref(a, tdeps, ignores) for a in ta) \
or not all(use_ref(a, rdeps, ignores) for a in ra):
continue
# Both dep lists should be equally long in case deps have just
# changed. If not, it means a dep has been added or removed and
# that means content change.
if len(tdeps) != len(rdeps):
continue
# If len is not different we still have to make sure that
# entries have the same pkg stem. The test above just saves time
# in some cases.
if not all(td in rdeps for td in tdeps):
continue
# Pkg only contains dependency change. Keep for further
# analysis.
if tdeps:
depend_changes[pfmri.get_pkg_stem(anarchy=True)] = tdeps
continue
# Pkg passed all checks and can be reversioned.
reversioned_pkgs.add(pfmri.get_pkg_stem(anarchy=True))
tracker.manifest_fetch_done()
def has_changed(pstem, seen=None, depth=0):
"""Determine if a package or any of its dependencies has
changed.
Function will check if a dependency had a content change. If it
only had a dependency change, analyze its dependencies
recursively. Only if the whole dependency chain didn't have any
content change it is safe to reversion the package.
Note about circular dependencies: The function keeps track of
pkgs it already processed by stuffing them into the set 'seen'.
However, 'seen' gets updated before the child dependencies of
the current pkg are examined. This works if 'seen' is only used
for one dependency chain since the function immediately comes
back with a True result if a pkg has changed further down the
tree. However, if 'seen' is re-used between runs, it will
return prematurely, likely returning wrong results. """
MAX_DEPTH = 100
if not seen:
seen = set()
if pstem in seen:
return False
depth += 1
if depth > MAX_DEPTH:
# Let's make sure we don't run into any
# recursion limits. If the dep chain is too deep
# just treat as changed pkg.
error(
_("Dependency chain depth of >{md:d} detected for"
" {p}.").format(md=MAX_DEPTH, p=p))
return True
# Pkg has no change at all.
if pstem in reversioned_pkgs:
return False
# Pkg must have content change, if it had no change it would be
# in reversioned_pkgs, and if it had just a dep change it would
# be in depend_changes.
if pstem not in depend_changes:
return True
# We need to update 'seen' here, otherwise we won't find this
# entry in case of a circular dependency.
seen.add(pstem)
return any(has_changed(d, seen, depth) for d in depend_changes[pstem])
# Check if packages which just have a dep change can be reversioned by
# checking if child dependencies also have no content change.
dep_revs = 0
for p in depend_changes:
if not has_changed(p):
dep_revs += 1
reversioned_pkgs.add(p)
status = []
if cmp_policy == CMP_UNSIGNED:
status.append((_("WARNING: Signature changes in file content "
"ignored in resurfacing")))
status.append((_("Packages to process:"), str(len(latest_pkgs))))
status.append((_("New packages:"), str(new_p)))
status.append((_("Unmodified packages:"), str(dups)))
if sucs:
# This only happens if reference repo is ahead of target repo,
# so only show if it actually happened.
status.append((_("Packages with successors in "
"reference repo:"), str(sucs)))
if nrevs:
# This only happens if user specified pkgs to not revert,
# so only show if it actually happened.
status.append((_("Packages not to be reversioned by user "
"request:"), str(nrevs)))
status.append((_("Packages with no content change:"),
str(len(reversioned_pkgs) - dep_revs)))
status.append((_("Packages which only have dependency change:"),
str(len(depend_changes))))
status.append(
(_("Packages with unchanged dependency chain:"), str(dep_revs)))
status.append(
(_("Packages to be reversioned:"), str(len(reversioned_pkgs))))
rjust_status = max(len(s[0]) for s in status)
rjust_value = max(len(s[1]) for s in status)
for s in status:
msg("{0} {1}".format(s[0].rjust(rjust_status),
s[1].rjust(rjust_value)))
if not reversioned_pkgs:
msg(_("\nNo packages to reversion."))
return False
if dry_run:
msg(_("\nReversioning packages (dry-run)."))
else:
msg(_("\nReversioning packages."))
# Start the main pass. Reversion packages from reversioned_pkgs to the
# version in the ref repo. For packages which don't get reversioned,
# check if the dependency versions are still correct, fix if necessary.
tracker.reversion_start(len(latest_pkgs), len(reversioned_pkgs))
for p in latest_pkgs:
tracker.reversion_add_progress(pfmri, pkgs=1)
modified = False
# Get the pkg fmri (pfmri) of the latest version based on if it
# has been reversioned or not.
stem = latest_pkgs[p].get_pkg_stem(anarchy=True)
if stem in reversioned_pkgs:
tracker.reversion_add_progress(pfmri, reversioned=1)
if dry_run:
continue
pfmri = latest_ref_pkgs[p]
# Retrieve manifest from ref repo and replace the one in
# the target repo. We don't have to adjust depndencies
# for these packages because they will not depend on
# anything we'll reversion.
rmani = ref_xport.get_manifest(pfmri)
if cmp_policy == CMP_UNSIGNED:
# Files with different signed content hash
# values can have equivalent unsigned content
# hash. CMP_UNSIGNED relaxes comparison
# constraints and allows this case to compare
# as equal. The reversioned manifest may
# reference file data that is not present in
# the target repository, so ensure that any
# missing file data is added to the target
# repository.
add_missing_files(target_repo, pub, latest_pkgs[p], pfmri,
rmani, ref, ref_repo, ref_xport,
ref_xport_cfg, ref_pub)
opath = target_repo.manifest(latest_pkgs[p], pub)
os.remove(opath)
path = target_repo.manifest(pfmri, pub)
try:
repo_modified = True
repo_finished = False
portable.rename(rmani.pathname, path)
except OSError as e:
abort(err=_("Could not reversion manifest "
"{path}: {err}").format(path=path, err=str(e)))
continue
# For packages we don't reversion we have to check if they
# depend on a reversioned package.
# Since the version of this dependency might be removed from the
# repo, we have to adjust the dep version to the one of the
# reversioned pkg.
pfmri = latest_pkgs[p]
omani = get_manifest(target_repo, pub, pfmri)
mani = manifest.Manifest(pfmri)
for act in omani.gen_actions():
nact = adjust_dep_action(p, act, latest_ref_pkgs, reversioned_pkgs,
ref_xport)
if nact:
mani.add_action(nact, misc.EmptyI)
if nact is not act:
modified = True
# Only touch manifest if something actually changed.
if modified:
tracker.reversion_add_progress(pfmri, adjusted=1)
if not dry_run:
path = target_repo.manifest(pfmri, pub)
repo_modified = True
repo_finished = False
mani.store(path)
tracker.reversion_done()
return True
def test_done(self):
"""Test that done() works properly."""
pi = progress.GoalTrackerItem("testitem")
pi.goalitems = 1
self.assertRaises(AssertionError, pi.done)
pi.done(goalcheck=False)
