def test_set_content1(self):
""" ASSERT: manifest string repr reflects its construction """
self.m1.set_content(self.m1_contents)
# It would be nice if we could just see if the string
# representation of the manifest matched the input, but the
# order of individual fields seems to change. Instead we look
# for useful substrings.
# Index raises an exception if the substring isn't found;
# if that were to happen, the test case would then fail.
str(self.m1).index("fmri=pkg:/library/libc")
str(self.m1).index("owner=sch")
str(self.m1).index("group=staff")
str(self.m1).index("isa=i386")
# Verify set_content with a byte string with unicode data
# works.
bstr = "set name=pkg.summary:th value=\"ซอฟต์แวร์ \""
m = manifest.Manifest()
m.set_content(bstr)
output = list(m.as_lines())[0].rstrip()
self.assertEqual(bstr, output)
self.assert_(isinstance(output, str))
# Verify set_content with a Unicode string results in a
# byte string (for now).
m = manifest.Manifest()
m.set_content(unicode(bstr, "utf-8"))
output = list(m.as_lines())[0].rstrip()
self.assertEqual(bstr, output)
self.assert_(isinstance(output, str))
def test_validate(self):
"""Verifies that Manifest validation works as expected."""
self.m2.set_content(self.m2_contents, signatures=True)
self.m2.validate(signatures=self.m2_signatures)
self.m2.set_content(self.diverse_contents, signatures=True)
self.assertRaises(api_errors.BadManifestSignatures,
self.m2.validate,
signatures=self.m2_signatures)
# Verify a manifest that has its content set using a byte string
# has the same signature as that of one set with a Unicode
# string when the content is the same.
bstr = "set name=pkg.summary:th value=\"ซอฟต์แวร์ \""
m1 = manifest.Manifest()
m1.set_content(bstr, signatures=True)
output1 = "".join(m1.as_lines())
m2 = manifest.Manifest()
if six.PY2:
m2.set_content(six.text_type(bstr, "utf-8"), signatures=True)
else:
m2.set_content(bstr, signatures=True)
output2 = "".join(m2.as_lines())
self.assertEqualDiff(output1, output2)
self.assertEqualDiff(m1.signatures, m2.signatures)
def setUp(self):
pkg5unittest.Pkg5TestCase.setUp(self)
self.m1 = manifest.Manifest()
self.m1_contents = """\
set com.sun,test=true
depend type=require fmri=pkg:/library/libc
file fff555fff mode=0555 owner=sch group=staff path=/usr/bin/i386/sort isa=i386
"""
self.m2 = manifest.Manifest()
self.m2_contents = """\
set com.sun,test=false
set com.sun,data=true
depend type=require fmri=pkg:/library/libc
file fff555ff9 mode=0555 owner=sch group=staff \\
path=/usr/bin/i386/sort isa=i386
file eeeaaaeee mode=0555 owner=sch group=staff path=/usr/bin/amd64/sort isa=amd64
file ff555fff mode=0555 owner=root group=bin path=/kernel/drv/foo isa=i386
file ff555ffe mode=0555 owner=root group=bin path=/kernel/drv/amd64/foo isa=amd64
file ff555ffd mode=0644 owner=root group=bin path=/kernel/drv/foo.conf
"""
self.m2_signatures = {
"sha-1": "7272cb2461a8a4ccf958b7a7f13f3ae20cbb0212"
}
#
# Try to keep this up to date with one of
# every action type.
#
self.diverse_contents = """\
set com.sun,test=false
set name=pkg.description value="The Z Shell (zsh) is a Bourne-like shell " \\
"designed for interactive use, although it is also a powerful scripting " \\
"language. Many of the useful features of bash, ksh, and tcsh were " \\
"incorporated into zsh, but many original features were added."
depend type=require fmri=pkg:/library/libc
file fff555ff9 mode=0555 owner=sch group=staff path=/usr/bin/i386/sort isa=i386
dir owner=root path=usr/bin group=bin mode=0755 variant.arch=i386 variant.arch=sparc
dir owner=root path="opt/dir with spaces in value" group=bin mode=0755
dir owner=root path="opt/dir with " \\
"whitespaces " \\
"in value" group=bin mode=0755
link path=usr/lib/amd64/libjpeg.so \\
target=libjpeg.so.62.0.0
hardlink path=usr/bin/amd64/rksh93 target=ksh93 variant.opensolaris.zone=global
group groupname=testgroup gid=10
"""
self.m4_contents = """\
set com.sun,test=false
set com.sun,data=true
depend type=require fmri=pkg:/library/libc
file fff555ff9 mode=0555 owner=sch group=staff path=/usr/bin/i386/sort \\
isa=i386
"""
self.m5_contents = """\
def test_set_content1(self):
""" ASSERT: manifest string repr reflects its construction """
self.m1.set_content(self.m1_contents)
# It would be nice if we could just see if the string
# representation of the manifest matched the input, but the
# order of individual fields seems to change. Instead we look
# for useful substrings.
# Index raises an exception if the substring isn't found;
# if that were to happen, the test case would then fail.
mstr = str(self.m1)
mstr.index("fmri=pkg:/library/libc")
mstr.index("owner=sch")
mstr.index("group=staff")
mstr.index("isa=i386")
# Verify set_content with a byte string with unicode data
# works.
bstr = "set name=pkg.summary:th value=\"ซอฟต์แวร์ \""
m = manifest.Manifest()
m.set_content(bstr)
output = list(m.as_lines())[0].rstrip()
self.assertEqual(bstr, output)
self.assertTrue(isinstance(output, str))
# Verify set_content with a Unicode string works.
m = manifest.Manifest()
if six.PY2:
m.set_content(six.text_type(bstr, "utf-8"))
else:
m.set_content(bstr)
output = list(m.as_lines())[0].rstrip()
self.assertEqual(bstr, output)
self.assertTrue(isinstance(output, str))
# Verify Manifests using line continuation '\' are parsed as
# expected.
m = manifest.Manifest()
m.set_content(self.diverse_contents)
expected = sorted('''\
set name=com.sun,test value=false
set name=pkg.description value="The Z Shell (zsh) is a Bourne-like shell designed for interactive use, although it is also a powerful scripting language. Many of the useful features of bash, ksh, and tcsh were incorporated into zsh, but many original features were added."
depend fmri=pkg:/library/libc type=require
group gid=10 groupname=testgroup
dir group=bin mode=0755 owner=root path="opt/dir with spaces in value"
dir group=bin mode=0755 owner=root path="opt/dir with whitespaces in value"
dir group=bin mode=0755 owner=root path=usr/bin variant.arch=i386 variant.arch=sparc
file fff555ff9 group=staff isa=i386 mode=0555 owner=sch path=usr/bin/i386/sort
hardlink path=usr/bin/amd64/rksh93 target=ksh93 variant.opensolaris.zone=global
link path=usr/lib/amd64/libjpeg.so target=libjpeg.so.62.0.0
'''.splitlines())
actual = sorted(l.strip() for l in m.as_lines())
self.assertEqualDiff(expected, actual)
def setUp(self):
pkg5unittest.Pkg5TestCase.setUp(self)
self.m1 = manifest.Manifest()
self.m1_contents = """\
set com.sun,test=true
depend type=require fmri=pkg:/library/libc
file fff555fff mode=0555 owner=sch group=staff path=/usr/bin/i386/sort isa=i386
"""
self.m2 = manifest.Manifest()
self.m2_contents = """\
set com.sun,test=false
set com.sun,data=true
depend type=require fmri=pkg:/library/libc
file fff555ff9 mode=0555 owner=sch group=staff \
path=/usr/bin/i386/sort isa=i386
file eeeaaaeee mode=0555 owner=sch group=staff path=/usr/bin/amd64/sort isa=amd64
file ff555fff mode=0555 owner=root group=bin path=/kernel/drv/foo isa=i386
file ff555ffe mode=0555 owner=root group=bin path=/kernel/drv/amd64/foo isa=amd64
file ff555ffd mode=0644 owner=root group=bin path=/kernel/drv/foo.conf
"""
self.m2_signatures = {
"sha-1": "e600f5e48a838b11ed73fd4afedfc35638ab0bbf"
}
#
# Try to keep this up to date with on of
# every action type.
#
self.diverse_contents = """\
set com.sun,test=false
depend type=require fmri=pkg:/library/libc
file fff555ff9 mode=0555 owner=sch group=staff path=/usr/bin/i386/sort isa=i386
dir owner=root path=usr/bin group=bin mode=0755 variant.arch=i386 variant.arch=sparc
dir owner=root path="opt/dir with spaces in value" group=bin mode=0755
dir owner=root path="opt/dir with whitespaces in value" group=bin mode=0755
link path=usr/lib/amd64/libjpeg.so \
target=libjpeg.so.62.0.0
hardlink path=usr/bin/amd64/rksh93 target=ksh93 variant.opensolaris.zone=global
group groupname=testgroup gid=10
"""
self.m4_contents = """\
set com.sun,test=false
set com.sun,data=true
depend type=require fmri=pkg:/library/libc
file fff555ff9 mode=0555 owner=sch group=staff path=/usr/bin/i386/sort \
isa=i386
"""
self.m5_contents = """\
def __do_alter_only_verify(self, pfmri, verbose=False, quiet=False, exit=0,
parsable=False, debug=""):
# Alter the owner, group, mode of all files (and
# directories) to something different than the package declares.
m = manifest.Manifest()
m.set_content(self.get_img_manifest(pfmri))
for a in m.gen_actions():
if a.name not in ("file", "dir"):
# Only want file or dir actions.
continue
ubin = portable.get_user_by_name("bin", None, False)
groot = portable.get_group_by_name("root", None, False)
fname = a.attrs["path"]
fpath = os.path.join(self.get_img_path(), fname)
os.chown(fpath, ubin, groot)
os.chmod(fpath, misc.PKG_RO_FILE_MODE)
verify_cmd = "verify"
if verbose:
verify_cmd += " -v"
if quiet:
verify_cmd += " -q"
if parsable:
verify_cmd += " --parsable=0"
self.pkg("{0}{1} {2}".format(debug, verify_cmd, pfmri),
exit=exit)
if exit == 0:
self.assertTrue("OK" in self.output and "ERROR" not
in self.output)
return exit
def __change_content_hash(self):
"""Change the content-hash attr in the manifest located at the
target and expected repos."""
mapping = {
self.dpath2: self.published_targ,
self.dpath3: self.published_exp
}
for repodir in (self.dpath2, self.dpath3):
for s in mapping[repodir]:
# Find elftest package
if "elftest" in s:
break
f = fmri.PkgFmri(s, None)
repo = self.get_repo(repodir)
mpath = repo.manifest(f)
# load manifest, change content-hash attr and store back
# to disk
mani = manifest.Manifest()
mani.set_content(pathname=mpath)
for a in mani.gen_actions():
if "bin/true" in str(a):
# change the signed version of hash of
# the ELF file
a.attrs["pkg.content-hash"][0] = "gelf:sha512t_256:foo"
mani.store(mpath)
# rebuild repo catalog since manifest digest changed
repo.rebuild()
def fetch_manifest(repouri, fmri):
"""Fetch the manifest for package-fmri 'fmri' from the server
in 'server_url'... return as Manifest object."""
mfst_str = xport.get_manifest(fmri, pub=repouri, content_only=True)
m = manifest.Manifest(fmri)
m.set_content(content=mfst_str)
return m
def addmanifest(self, root, mfile, arch, modechecks, exceptions):
"""Treats the specified input file as a pkg(5) package
manifest, and extends the ManifestTree dictionary with entries
for the actions therein.
"""
mfest = manifest.Manifest()
try:
mfest.set_content(open(os.path.join(root, mfile)).read())
except IOError, exc:
raise IOError("cannot read manifest: %s" % str(exc))
def make_manifest(fp):
"""Given the file path, 'fp', return a Manifest for that path."""
m = manifest.Manifest()
try:
fh = open(fp, "rb")
lines = fh.read()
fh.close()
except EnvironmentError, e:
raise
def get_manifest(repouri, fmri):
"""Fetch the manifest for package-fmri 'fmri' from the source
in 'repouri'... return as Manifest object."""
# support null manifests to keep lists ordered for merge
if not fmri:
return null_manifest
mfst_str = xport.get_manifest(fmri, pub=repouri, content_only=True)
m = manifest.Manifest(fmri)
m.set_content(content=mfst_str)
return m
def get_manifest(repo, pub, pfmri):
""" Retrieve a manifest with FMRI 'pfmri' of publisher 'pub' from
repository object 'repo'. """
path = repo.manifest(pfmri, pub)
mani = manifest.Manifest(pfmri)
try:
mani.set_content(pathname=path)
except Exception as e:
abort(err=_("Can not open manifest file {file}: {err}\n"
"Please run 'pkgrepo verify -s {rroot}' to check the "
"integrity of the repository.").format(
file=path, err=str(e), rroot=repo.root))
return mani
def test_diffs4(self):
""" ASSERT: Building m' from diff(m, null) should yield m """
self.m2.set_content(self.m2_contents)
diffs = self.m2.combined_difference(manifest.null)
new_contents = ""
for d in diffs:
new_contents += str(d[1]) + "\n"
mtmp = manifest.Manifest()
#print(new_contents)
mtmp.set_content(new_contents)
diffs = self.m2.combined_difference(mtmp)
self.assertEqual(len(diffs), 0)
def __make_manifest(fp, basedirs=None, load_data=True):
"""Given the file path, 'fp', return a Manifest for that path."""
m = manifest.Manifest()
fh = open(fp, "rb")
acts = []
missing_files = []
accumulate = ""
for l in fh:
l = l.strip()
if l.endswith("\\"):
accumulate += l[0:-1]
continue
elif accumulate:
l = accumulate + l
accumulate = ""
if not l or l[0] == '#':
continue
try:
a, local_path, used_bd = actions.internalizestr(l,
basedirs=basedirs,
load_data=load_data)
if local_path:
assert portable.PD_LOCAL_PATH not in a.attrs
a.attrs[portable.PD_LOCAL_PATH] = local_path
a.attrs[portable.PD_PROTO_DIR] = used_bd
a.attrs[portable.PD_PROTO_DIR_LIST] = basedirs
acts.append(a)
except actions.ActionDataError, e:
new_a, local_path, used_bd = actions.internalizestr(
l, basedirs=basedirs, load_data=False)
if new_a.name == "license":
acts.append(new_a)
else:
path = e.path
# If the path was not set, then parse the
# action, without trying to load the data and
# use the path defined in the action.
if not path:
path = new_a.attrs["path"]
missing_files.append(base.MissingFile(path))
def __merge_fmris(new_fmri, manifest_list, fmri_list, variant_list, variant):
"""Private merge implementation."""
# Remove variant tags, package variant metadata, and signatures
# from manifests since we're reassigning. This allows merging
# pre-tagged, already merged pkgs, or signed packages.
blended_actions = []
blend_names = set([variant, variant[8:]])
for j, m in enumerate(manifest_list):
deleted_count = 0
vval = variant_list[j]
for i, a in enumerate(m.actions[:]):
if a.name == "signature" or \
(a.name == "set" and a.attrs["name"] == "pkg.fmri"):
# signatures and pkg.fmri actions are no longer
# valid after merging
del m.actions[i - deleted_count]
deleted_count += 1
continue
if variant in a.attrs:
if a.attrs[variant] != vval:
# we have an already merged
# manifest; filter out actions
# for other variants
del m.actions[i - deleted_count]
deleted_count += 1
continue
else:
del a.attrs[variant]
if a.name == "set" and a.attrs["name"] == variant:
if vval not in a.attrlist("value"):
raise PkgmergeException(
_("package {pkg} is tagged as "
"not supporting {var_name} "
"{var_value}").format(
pkg=fmri_list[j],
var_name=variant,
var_value=vval))
del m.actions[i - deleted_count]
deleted_count += 1
# checking if we're supposed to blend this action
# for this variant. Handle prepended "variant.".
if blend_names & set(a.attrlist("pkg.merge.blend")):
blended_actions.append((j, a))
# add blended actions to other manifests
for j, m in enumerate(manifest_list):
for k, a in blended_actions:
if k != j:
m.actions.append(a)
# Like the unix utility comm, except that this function
# takes an arbitrary number of manifests and compares them,
# returning a tuple consisting of each manifest's actions
# that are not the same for all manifests, followed by a
# list of actions that are the same in each manifest.
try:
action_lists = list(manifest.Manifest.comm(manifest_list))
except manifest.ManifestError as e:
raise PkgmergeException(
"Duplicate action(s) in package \"{0}\": \n{1}".format(
new_fmri.pkg_name, e))
# Declare new package FMRI.
action_lists[-1].insert(0,
actions.fromstr("set name=pkg.fmri value={0}".format(new_fmri)))
for a_list, v in zip(action_lists[:-1], variant_list):
for a in a_list:
a.attrs[variant] = v
# discard any blend tags for this variant from common list
for a in action_lists[-1]:
blend_attrs = set(a.attrlist("pkg.merge.blend"))
match = blend_names & blend_attrs
for m in list(match):
if len(blend_attrs) == 1:
del a.attrs["pkg.merge.blend"]
else:
a.attrlist("pkg.merge.blend").remove(m)
# combine actions into single list
allactions = reduce(lambda a, b: a + b, action_lists)
# figure out which variants are actually there for this pkg
actual_variant_list = [
v
for m, v in zip(manifest_list, variant_list)
]
# add set action to document which variants are supported
allactions.append(actions.fromstr("set name={0} {1}".format(variant,
" ".join([
"value={0}".format(a)
for a in actual_variant_list
])
)))
allactions.sort()
m = manifest.Manifest(pfmri=new_fmri)
m.set_content(content=allactions)
return m
def test_4_unsigned_option(self):
self.__change_content_hash()
# Copy target repo to tmp repo
self.copy_repository(self.dpath2, self.dpath_tmp,
{"selachii": "selachii"})
# The new repository won't have a catalog, so rebuild it.
self.dcs[4].get_repo(auto_create=True).rebuild()
# If '-u' is enabled, we just check the unsigned version of
# hashes so that the target basic package is treated as no
# content change and should be reversioned.
elftest_exp = self.elftest_ref
efldiff_exp = self.elfdiff_ref
# Replace elftest package in the expected repo.
for i, s in enumerate(self.published_exp):
if "elftest" in s:
self.published_exp[i] = self.pkgsend_bulk(
self.dpath3, (elftest_exp, ))[0]
if "elfdiff" in s:
self.published_exp[i] = self.pkgsend_bulk(
self.dpath3, (efldiff_exp, ))[0]
# Check that '-u' option works and should not affect other
# packages.
self.pkgsurf("-s {0} -r {1} -u".format(self.dpath_tmp, self.dpath1))
self.pkgrepo("-s {0} verify --disable dependency".format(
self.dpath_tmp))
# Create a target repo that just contains elfdiff and elfshare
# package. Ultimately the file elftest.so.1 and elftest.so.2
# should reside in the target repo.
self.pkgsend_bulk(self.dpath5, [self.elfdiff_targ, self.elfshare_targ])
self.pkgsurf("-s {0} -r {1} -u".format(self.dpath5, self.dpath1))
self.pkgrepo("-s {0} verify --disable dependency".format(self.dpath5))
# Test the HTTP-based reference repo case.
self.pkgsend_bulk(self.dpath6, [self.elfdiff_targ, self.elfshare_targ])
self.pkgsurf("-s {0} -r {1} -u".format(self.dpath6, self.durl1))
self.pkgrepo("-s {0} verify --disable dependency".format(self.dpath6))
ref_repo = self.get_repo(self.dpath1)
targ_repo = self.get_repo(self.dpath_tmp)
exp_repo = self.get_repo(self.dpath3)
for s in self.published_exp:
f = fmri.PkgFmri(s, None)
targ = targ_repo.manifest(f)
# Load target manifest
targm = manifest.Manifest()
targm.set_content(pathname=targ)
# Load expected manifest
exp = exp_repo.manifest(f)
expm = manifest.Manifest()
expm.set_content(pathname=exp)
ta, ra, ca = manifest.Manifest.comm([targm, expm],
cmp_policy=CMP_UNSIGNED)
self.debug("{0}: {1:d} {2:d}".format(str(s), len(ta), len(ra)))
self.assertEqual(
0, len(ta), "{0} had unexpected actions:"
" \n{1}".format(s, "\n".join([str(x) for x in ta])))
self.assertEqual(
0, len(ra), "{0} had missing actions: "
"\n{1}".format(s, "\n".join([str(x) for x in ra])))
sys.exit(EXIT_OOPS)
class PkgmergeException(Exception):
"""An exception raised if something goes wrong during the merging
process."""
def __unicode__(self):
# To workaround python issues 6108 and 2517, this provides a
# a standard wrapper for this class' exceptions so that they
# have a chance of being stringified correctly.
return str(self)
catalog_dict = {} # hash table of catalogs by source uri
fmri_cache = {}
manifest_cache = {}
null_manifest = manifest.Manifest()
tmpdir = None
dry_run = False
xport = None
dest_xport = None
pubs = set()
target_pub = None
def cleanup():
"""To be called at program finish."""
if tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
if dry_run:
return
def main_func():
gettext.install("pkg",
"/usr/share/locale",
codeset=locale.getpreferredencoding())
ignoreattrs = []
onlyattrs = []
onlytypes = []
varattrs = defaultdict(set)
try:
opts, pargs = getopt.getopt(sys.argv[1:], "i:o:t:v:?", ["help"])
for opt, arg in opts:
if opt == "-i":
ignoreattrs.append(arg)
elif opt == "-o":
onlyattrs.append(arg)
elif opt == "-t":
onlytypes.extend(arg.split(","))
elif opt == "-v":
args = arg.split("=")
if len(args) != 2:
usage(_("variant option incorrect {0}").format(arg))
if not args[0].startswith("variant."):
args[0] = "variant." + args[0]
varattrs[args[0]].add(args[1])
elif opt in ("--help", "-?"):
usage(exitcode=0)
except getopt.GetoptError as e:
usage(_("illegal global option -- {0}").format(e.opt))
if len(pargs) != 2:
usage(_("two manifest arguments are required"))
if (pargs[0] == "-" and pargs[1] == "-"):
usage(_("only one manifest argument can be stdin"))
if ignoreattrs and onlyattrs:
usage(_("-i and -o options may not be used at the same time."))
for v in varattrs:
if len(varattrs[v]) > 1:
usage(_("For any variant, only one value may be " "specified."))
varattrs[v] = varattrs[v].pop()
ignoreattrs = set(ignoreattrs)
onlyattrs = set(onlyattrs)
onlytypes = set(onlytypes)
utypes = set(t for t in onlytypes
if t == "generic" or t not in pkg.actions.types)
if utypes:
usage(
_("unknown action types: {0}".format(apx.list_to_lang(
list(utypes)))))
manifest1 = manifest.Manifest()
manifest2 = manifest.Manifest()
try:
# This assumes that both pargs are not '-'.
for p, m in zip(pargs, (manifest1, manifest2)):
if p == "-":
m.set_content(content=sys.stdin.read())
else:
m.set_content(pathname=p)
except (pkg.actions.ActionError, apx.InvalidPackageErrors) as e:
error(_("Action error in file {p}: {e}").format(**locals()))
except (EnvironmentError, apx.ApiException) as e:
error(e)
#
# manifest filtering
#
# filter action type
if onlytypes:
for m in (manifest1, manifest2):
# Must pass complete list of actions to set_content, not
# a generator, to avoid clobbering manifest contents.
m.set_content(content=list(m.gen_actions_by_types(onlytypes)))
# filter variant
v1 = manifest1.get_all_variants()
v2 = manifest2.get_all_variants()
for vname in varattrs:
for _path, v, m in zip(pargs, (v1, v2), (manifest1, manifest2)):
if vname not in v:
continue
filt = varattrs[vname]
if filt not in v[vname]:
usage(
_("Manifest {path} doesn't support "
"variant {vname}={filt}".format(**locals())))
# remove the variant tag
def rip(a):
a.attrs.pop(vname, None)
return a
m.set_content([
rip(a) for a in m.gen_actions(
excludes=[variant.Variants({
vname: filt
}).allow_action])
])
m[vname] = filt
if varattrs:
# need to rebuild these if we're filtering variants
v1 = manifest1.get_all_variants()
v2 = manifest2.get_all_variants()
# we need to be a little clever about variants, since
# we can have multiple actions w/ the same key attributes
# in each manifest in that case. First, make sure any variants
# of the same name have the same values defined.
for k in set(v1.keys()) & set(v2.keys()):
if v1[k] != v2[k]:
error(
_("Manifests support different variants "
"{v1} {v2}").format(v1=v1, v2=v2))
# Now, get a list of all possible variant values, including None
# across all variants and both manifests
v_values = dict()
for v in v1:
v1[v].add(None)
for a in v1[v]:
v_values.setdefault(v, set()).add((v, a))
for v in v2:
v2[v].add(None)
for a in v2[v]:
v_values.setdefault(v, set()).add((v, a))
diffs = []
for tup in product(*v_values.values()):
# build excludes closure to examine only actions exactly
# matching current variant values... this is needed to
# avoid confusing manifest difference code w/ multiple
# actions w/ same key attribute values or getting dups
# in output
def allow(a, publisher=None):
for k, v in tup:
if v is not None:
if k not in a.attrs or a.attrs[k] != v:
return False
elif k in a.attrs:
return False
return True
a, c, r = manifest2.difference(manifest1, [allow], [allow])
diffs += a
diffs += c
diffs += r
# License action still causes spurious diffs... check again for now.
real_diffs = [(a, b) for a, b in diffs
if a is None or b is None or a.different(b)]
if not real_diffs:
return 0
# define some ordering functions so that output is easily readable
# First, a human version of action comparison that works across
# variants and action changes...
def compare(a, b):
if hasattr(a, "key_attr") and hasattr(b, "key_attr") and \
a.key_attr == b.key_attr:
res = cmp(a.attrs[a.key_attr], b.attrs[b.key_attr])
if res:
return res
# sort by variant
res = cmp(sorted(list(a.get_variant_template())),
sorted(list(b.get_variant_template())))
if res:
return res
else:
res = cmp(a.ordinality, b.ordinality)
if res:
return res
return cmp(str(a), str(b))
# and something to pull the relevant action out of the old value, new
# value tuples
def tuple_key(a):
if not a[0]:
return a[1]
return a[0]
# sort and....
diffs = sorted(diffs, key=tuple_key, cmp=compare)
# handle list attributes
def attrval(attrs, k, elide_iter=tuple()):
def q(s):
if " " in s or s == "":
return '"{0}"'.format(s)
else:
return s
v = attrs[k]
if isinstance(v, list) or isinstance(v, set):
out = " ".join([
"{0}={1}".format(k, q(lmt)) for lmt in sorted(v)
if lmt not in elide_iter
])
elif " " in v or v == "":
out = k + "=\"" + v + "\""
else:
out = k + "=" + v
return out
# figure out when to print diffs
def conditional_print(s, a):
if onlyattrs:
if not set(a.attrs.keys()) & onlyattrs:
return False
elif ignoreattrs:
if not set(a.attrs.keys()) - ignoreattrs:
return False
print("{0} {1}".format(s, a))
return True
different = False
for old, new in diffs:
if not new:
different |= conditional_print("-", old)
elif not old:
different |= conditional_print("+", new)
else:
s = []
if not onlyattrs:
if (hasattr(old, "hash") and "hash" not in ignoreattrs):
if old.hash != new.hash:
s.append(" - {0}".format(old.hash))
s.append(" + {0}".format(new.hash))
attrdiffs = (set(new.differences(old)) - ignoreattrs)
attrsames = sorted(
list(
set(old.attrs.keys() + new.attrs.keys()) -
set(new.differences(old))))
else:
if hasattr(old, "hash") and "hash" in onlyattrs:
if old.hash != new.hash:
s.append(" - {0}".format(old.hash))
s.append(" + {0}".format(new.hash))
attrdiffs = (set(new.differences(old)) & onlyattrs)
attrsames = sorted(
list(
set(old.attrs.keys() + new.attrs.keys()) -
set(new.differences(old))))
for a in sorted(attrdiffs):
if a in old.attrs and a in new.attrs and \
isinstance(old.attrs[a], list) and \
isinstance(new.attrs[a], list):
elide_set = (set(old.attrs[a]) & set(new.attrs[a]))
else:
elide_set = set()
if a in old.attrs:
diff_str = attrval(old.attrs, a, elide_iter=elide_set)
if diff_str:
s.append(" - {0}".format(diff_str))
if a in new.attrs:
diff_str = attrval(new.attrs, a, elide_iter=elide_set)
if diff_str:
s.append(" + {0}".format(diff_str))
# print out part of action that is the same
if s:
different = True
print("{0} {1} {2}".format(
old.name, attrval(old.attrs, old.key_attr), " ".join(
("{0}".format(attrval(old.attrs, v)) for v in attrsames
if v != old.key_attr))))
for l in s:
print(l)
return int(different)
def info(self, fmri_strings, info_needed, excludes=misc.EmptyI):
"""Gathers information about fmris. fmri_strings is a list
of fmri_names for which information is desired. It
returns a dictionary of lists. The keys for the dictionary are
the constants specified in the class definition. The values are
lists of PackageInfo objects or strings."""
bad_opts = info_needed - PackageInfo.ALL_OPTIONS
if bad_opts:
raise api_errors.UnrecognizedOptionsToInfo(bad_opts)
fmris = []
notfound = []
illegals = []
for pattern in fmri_strings:
try:
pfmri = None
pfmri = self.get_matching_pattern_fmris(pattern)
except pkg.fmri.IllegalFmri as e:
illegals.append(pattern)
continue
else:
fmris.extend(pfmri[0])
if not pfmri:
notfound.append(pattern)
repo_cat = self._depot.repo.get_catalog(self._pub)
# Set of options that can use catalog data.
cat_opts = frozenset([PackageInfo.SUMMARY,
PackageInfo.CATEGORIES, PackageInfo.DESCRIPTION,
PackageInfo.DEPENDENCIES])
# Set of options that require manifest retrieval.
act_opts = PackageInfo.ACTION_OPTIONS - \
frozenset([PackageInfo.DEPENDENCIES])
pis = []
for f in fmris:
pub = name = version = release = None
build_release = branch = packaging_date = None
if PackageInfo.IDENTITY in info_needed:
pub, name, version = f.tuple()
release = version.release
build_release = version.build_release
branch = version.branch
packaging_date = \
version.get_timestamp().strftime("%c")
states = None
links = hardlinks = files = dirs = dependencies = None
summary = csize = size = licenses = cat_info = \
description = None
if cat_opts & info_needed:
summary, description, cat_info, dependencies = \
_get_pkg_cat_data(repo_cat, info_needed,
excludes=excludes, pfmri=f)
if cat_info is not None:
cat_info = [
PackageCategory(scheme, cat)
for scheme, cat in cat_info
]
if (frozenset([PackageInfo.SIZE,
PackageInfo.LICENSES]) | act_opts) & info_needed:
mfst = manifest.Manifest(f)
try:
mpath = self._depot.repo.manifest(f)
except srepo.RepositoryError as e:
notfound.append(f)
continue
if not os.path.exists(mpath):
notfound.append(f)
continue
mfst.set_content(pathname=mpath)
if PackageInfo.LICENSES in info_needed:
licenses = self.__licenses(mfst)
if PackageInfo.SIZE in info_needed:
size, csize = mfst.get_size(
excludes=excludes)
if act_opts & info_needed:
if PackageInfo.LINKS in info_needed:
links = list(
mfst.gen_key_attribute_value_by_type(
"link", excludes))
if PackageInfo.HARDLINKS in info_needed:
hardlinks = list(
mfst.gen_key_attribute_value_by_type(
"hardlink", excludes))
if PackageInfo.FILES in info_needed:
files = list(
mfst.gen_key_attribute_value_by_type(
"file", excludes))
if PackageInfo.DIRS in info_needed:
dirs = list(
mfst.gen_key_attribute_value_by_type(
"dir", excludes))
pis.append(PackageInfo(pkg_stem=name, summary=summary,
category_info_list=cat_info, states=states,
publisher=pub, version=release,
build_release=build_release, branch=branch,
packaging_date=packaging_date, size=size,
csize=csize, pfmri=f, licenses=licenses,
links=links, hardlinks=hardlinks, files=files,
dirs=dirs, dependencies=dependencies,
description=description))
return {
self.INFO_FOUND: pis,
self.INFO_MISSING: notfound,
self.INFO_ILLEGALS: illegals
}
def do_reversion(pub, ref_pub, target_repo, ref_xport, changes, ignores,
cmp_policy, ref_repo, ref, ref_xport_cfg):
"""Do the repo reversion.
Return 'True' if repo got modified, 'False' otherwise."""
global temp_root, tracker, dry_run, repo_finished, repo_modified
target_cat = target_repo.get_catalog(pub=pub)
ref_cat = fetch_catalog(ref_pub, ref_xport, temp_root)
latest_pkgs = get_latest(target_cat)
latest_ref_pkgs = get_latest(ref_cat)
no_revs = get_matching_pkgs(target_cat, changes)
# We use bulk prefetching for faster transport of the manifests.
# Prefetch requires an intent which it sends to the server. Here
# we just use operation=reversion for all FMRIs.
intent = "operation=reversion;"
# use list() to force the zip() to evaluate
ref_pkgs = list(zip(latest_ref_pkgs.values(), repeat(intent)))
# Retrieve reference manifests.
# Try prefetching manifests in bulk first for faster, parallel
# transport. Retryable errors during prefetch are ignored and
# manifests are retrieved again during the "Reading" phase.
ref_xport.prefetch_manifests(ref_pkgs, progtrack=tracker)
# Need to change the output of mfst_fetch since otherwise we
# would see "Download Manifests x/y" twice, once from the
# prefetch and once from the actual manifest analysis.
tracker.mfst_fetch = progress.GoalTrackerItem(_("Analyzing Manifests"))
tracker.manifest_fetch_start(len(latest_pkgs))
reversioned_pkgs = set()
depend_changes = {}
dups = 0 # target pkg has equal version to ref pkg
new_p = 0 # target pkg not in ref
sucs = 0 # ref pkg is successor to pkg in targ
nrevs = 0 # pkgs requested to not be reversioned by user
for p in latest_pkgs:
# First check if the package is in the list of FMRIs the user
# doesn't want to reversion.
if p in no_revs:
nrevs += 1
tracker.manifest_fetch_progress(completion=True)
continue
# Check if the package is in the ref repo, if not: ignore.
if not p in latest_ref_pkgs:
new_p += 1
tracker.manifest_fetch_progress(completion=True)
continue
pfmri = latest_pkgs[p]
# Ignore if latest package is the same in targ and ref.
if pfmri == latest_ref_pkgs[p]:
dups += 1
tracker.manifest_fetch_progress(completion=True)
continue
# Ignore packages where ref version is higher.
if latest_ref_pkgs[p].is_successor(pfmri):
sucs += 1
tracker.manifest_fetch_progress(completion=True)
continue
# Pull the manifests for target and ref repo.
dm = get_manifest(target_repo, pub, pfmri)
rm = ref_xport.get_manifest(latest_ref_pkgs[p])
tracker.manifest_fetch_progress(completion=True)
tdeps = set()
rdeps = set()
# Diff target and ref manifest.
# action only in targ, action only in ref, common action
ta, ra, ca = manifest.Manifest.comm([dm, rm], cmp_policy=cmp_policy)
# Check for manifest changes.
if not all(use_ref(a, tdeps, ignores) for a in ta) \
or not all(use_ref(a, rdeps, ignores) for a in ra):
continue
# Both dep lists should be equally long in case deps have just
# changed. If not, it means a dep has been added or removed and
# that means content change.
if len(tdeps) != len(rdeps):
continue
# If len is not different we still have to make sure that
# entries have the same pkg stem. The test above just saves time
# in some cases.
if not all(td in rdeps for td in tdeps):
continue
# Pkg only contains dependency change. Keep for further
# analysis.
if tdeps:
depend_changes[pfmri.get_pkg_stem(anarchy=True)] = tdeps
continue
# Pkg passed all checks and can be reversioned.
reversioned_pkgs.add(pfmri.get_pkg_stem(anarchy=True))
tracker.manifest_fetch_done()
def has_changed(pstem, seen=None, depth=0):
"""Determine if a package or any of its dependencies has
changed.
Function will check if a dependency had a content change. If it
only had a dependency change, analyze its dependencies
recursively. Only if the whole dependency chain didn't have any
content change it is safe to reversion the package.
Note about circular dependencies: The function keeps track of
pkgs it already processed by stuffing them into the set 'seen'.
However, 'seen' gets updated before the child dependencies of
the current pkg are examined. This works if 'seen' is only used
for one dependency chain since the function immediately comes
back with a True result if a pkg has changed further down the
tree. However, if 'seen' is re-used between runs, it will
return prematurely, likely returning wrong results. """
MAX_DEPTH = 100
if not seen:
seen = set()
if pstem in seen:
return False
depth += 1
if depth > MAX_DEPTH:
# Let's make sure we don't run into any
# recursion limits. If the dep chain is too deep
# just treat as changed pkg.
error(
_("Dependency chain depth of >{md:d} detected for"
" {p}.").format(md=MAX_DEPTH, p=p))
return True
# Pkg has no change at all.
if pstem in reversioned_pkgs:
return False
# Pkg must have content change, if it had no change it would be
# in reversioned_pkgs, and if it had just a dep change it would
# be in depend_changes.
if pstem not in depend_changes:
return True
# We need to update 'seen' here, otherwise we won't find this
# entry in case of a circular dependency.
seen.add(pstem)
return any(has_changed(d, seen, depth) for d in depend_changes[pstem])
# Check if packages which just have a dep change can be reversioned by
# checking if child dependencies also have no content change.
dep_revs = 0
for p in depend_changes:
if not has_changed(p):
dep_revs += 1
reversioned_pkgs.add(p)
status = []
if cmp_policy == CMP_UNSIGNED:
status.append((_("WARNING: Signature changes in file content "
"ignored in resurfacing")))
status.append((_("Packages to process:"), str(len(latest_pkgs))))
status.append((_("New packages:"), str(new_p)))
status.append((_("Unmodified packages:"), str(dups)))
if sucs:
# This only happens if reference repo is ahead of target repo,
# so only show if it actually happened.
status.append((_("Packages with successors in "
"reference repo:"), str(sucs)))
if nrevs:
# This only happens if user specified pkgs to not revert,
# so only show if it actually happened.
status.append((_("Packages not to be reversioned by user "
"request:"), str(nrevs)))
status.append((_("Packages with no content change:"),
str(len(reversioned_pkgs) - dep_revs)))
status.append((_("Packages which only have dependency change:"),
str(len(depend_changes))))
status.append(
(_("Packages with unchanged dependency chain:"), str(dep_revs)))
status.append(
(_("Packages to be reversioned:"), str(len(reversioned_pkgs))))
rjust_status = max(len(s[0]) for s in status)
rjust_value = max(len(s[1]) for s in status)
for s in status:
msg("{0} {1}".format(s[0].rjust(rjust_status),
s[1].rjust(rjust_value)))
if not reversioned_pkgs:
msg(_("\nNo packages to reversion."))
return False
if dry_run:
msg(_("\nReversioning packages (dry-run)."))
else:
msg(_("\nReversioning packages."))
# Start the main pass. Reversion packages from reversioned_pkgs to the
# version in the ref repo. For packages which don't get reversioned,
# check if the dependency versions are still correct, fix if necessary.
tracker.reversion_start(len(latest_pkgs), len(reversioned_pkgs))
for p in latest_pkgs:
tracker.reversion_add_progress(pfmri, pkgs=1)
modified = False
# Get the pkg fmri (pfmri) of the latest version based on if it
# has been reversioned or not.
stem = latest_pkgs[p].get_pkg_stem(anarchy=True)
if stem in reversioned_pkgs:
tracker.reversion_add_progress(pfmri, reversioned=1)
if dry_run:
continue
pfmri = latest_ref_pkgs[p]
# Retrieve manifest from ref repo and replace the one in
# the target repo. We don't have to adjust depndencies
# for these packages because they will not depend on
# anything we'll reversion.
rmani = ref_xport.get_manifest(pfmri)
if cmp_policy == CMP_UNSIGNED:
# Files with different signed content hash
# values can have equivalent unsigned content
# hash. CMP_UNSIGNED relaxes comparison
# constraints and allows this case to compare
# as equal. The reversioned manifest may
# reference file data that is not present in
# the target repository, so ensure that any
# missing file data is added to the target
# repository.
add_missing_files(target_repo, pub, latest_pkgs[p], pfmri,
rmani, ref, ref_repo, ref_xport,
ref_xport_cfg, ref_pub)
opath = target_repo.manifest(latest_pkgs[p], pub)
os.remove(opath)
path = target_repo.manifest(pfmri, pub)
try:
repo_modified = True
repo_finished = False
portable.rename(rmani.pathname, path)
except OSError as e:
abort(err=_("Could not reversion manifest "
"{path}: {err}").format(path=path, err=str(e)))
continue
# For packages we don't reversion we have to check if they
# depend on a reversioned package.
# Since the version of this dependency might be removed from the
# repo, we have to adjust the dep version to the one of the
# reversioned pkg.
pfmri = latest_pkgs[p]
omani = get_manifest(target_repo, pub, pfmri)
mani = manifest.Manifest(pfmri)
for act in omani.gen_actions():
nact = adjust_dep_action(p, act, latest_ref_pkgs, reversioned_pkgs,
ref_xport)
if nact:
mani.add_action(nact, misc.EmptyI)
if nact is not act:
modified = True
# Only touch manifest if something actually changed.
if modified:
tracker.reversion_add_progress(pfmri, adjusted=1)
if not dry_run:
path = target_repo.manifest(pfmri, pub)
repo_modified = True
repo_finished = False
mani.store(path)
tracker.reversion_done()
return True
def test_1_basics(self):
"""Test basic resurfacing operation."""
# Copy target repo to tmp repo
self.copy_repository(self.dpath2, self.dpath_tmp,
{"selachii": "selachii"})
# The new repository won't have a catalog, so rebuild it.
self.dcs[4].get_repo(auto_create=True).rebuild()
#self.assertTrue(False)
# Check that empty repos get handled correctly
tempdir = tempfile.mkdtemp(dir=self.test_root)
# No repo at all
self.pkgsurf("-s {0} -r {1}".format(tempdir, self.dpath1), exit=1)
self.pkgsurf("-s {0} -r {1}".format(self.dpath1, tempdir), exit=1)
# Repo empty
self.pkgrepo("create -s {0}".format(tempdir))
self.pkgsurf("-s {0} -r {1}".format(tempdir, self.dpath1), exit=1)
self.pkgsurf("-s {0} -r {1}".format(self.dpath1, tempdir), exit=1)
# No packages
self.pkgrepo("add-publisher -s {0} selachii".format(tempdir))
self.pkgsurf("-s {0} -r {1}".format(tempdir, self.dpath1))
self.assertTrue("No packages to reversion." in self.output)
self.pkgsurf("-s {0} -r {1}".format(self.dpath1, tempdir))
self.assertTrue("No packages to reversion." in self.output)
shutil.rmtree(tempdir)
# Now check if it actually works.
self.pkgsurf("-s {0} -r {1}".format(self.dpath_tmp, self.dpath1))
ref_repo = self.get_repo(self.dpath1)
targ_repo = self.get_repo(self.dpath_tmp)
exp_repo = self.get_repo(self.dpath3)
for s in self.published_exp:
f = fmri.PkgFmri(s, None)
targ = targ_repo.manifest(f)
# Load target manifest
targm = manifest.Manifest()
targm.set_content(pathname=targ)
# Load expected manifest
exp = exp_repo.manifest(f)
expm = manifest.Manifest()
expm.set_content(pathname=exp)
ta, ra, ca = manifest.Manifest.comm([targm, expm])
self.debug("{0}: {1:d} {2:d}".format(str(s), len(ta), len(ra)))
self.assertEqual(
0, len(ta), "{0} had unexpected actions:"
" \n{1}".format(s, "\n".join([str(x) for x in ta])))
self.assertEqual(
0, len(ra), "{0} had missing actions: "
"\n{1}".format(s, "\n".join([str(x) for x in ra])))
# Check that pkgsurf informed the user that there is a newer
# version of a pkg in the ref repo.
self.assertTrue("Packages with successors" in self.output)
# Check that ignore option works.
# Just run again and see if goblin pkg now gets reversioned.
self.pkgsurf("-s {0} -r {1} -i info.home".format(
self.dpath_tmp, self.dpath1))
# Find goblin package
for s in self.published_ref:
if "goblin" in s:
break
f = fmri.PkgFmri(s, None)
targ = targ_repo.manifest(f)
ref = ref_repo.manifest(f)
self.assertEqual(
misc.get_data_digest(targ, hash_func=digest.DEFAULT_HASH_FUNC),
misc.get_data_digest(ref, hash_func=digest.DEFAULT_HASH_FUNC))
# Check that running the tool again doesn't find any pkgs
# to reversion. Use http for accessing reference repo this time.
self.pkgsurf("-s {0} -r {1}".format(self.dpath_tmp, self.durl1))
self.assertTrue("No packages to reversion." in self.output)
class CatalogInterface(_Interface):
"""This class presents an interface to server catalog objects that
clients may use.
"""
# Constants used to reference specific values that info can return.
INFO_FOUND = 0
INFO_MISSING = 1
INFO_ILLEGALS = 3
def fmris(self):
"""A generator function that produces FMRIs as it iterates
over the contents of the server's catalog."""
try:
c = self._depot.repo.catalog
except srepo.RepositoryMirrorError:
return iter(())
return self._depot.repo.catalog.fmris()
def get_entry_all_variants(self, pfmri):
"""A generator function that yields tuples of the format
(var_name, variants); where var_name is the name of the
variant and variants is a list of the variants for that
name."""
try:
c = self._depot.repo.catalog
except srepo.RepositoryMirrorError:
return iter(((), {}))
return self._depot.repo.catalog.get_entry_all_variants(pfmri)
def get_matching_pattern_fmris(self, patterns):
"""Returns a tuple of a sorted list of PkgFmri objects, newest
versions first, for packages matching those found in the
'patterns' list, and a dict of unmatched patterns indexed by
match criteria.
"""
try:
c = self._depot.repo.catalog
except srepo.RepositoryMirrorError:
return tuple(), {}
return pkg.catalog.extract_matching_fmris(c.fmris(),
patterns=patterns)
def get_matching_version_fmris(self, versions):
"""Returns a tuple of a sorted list of PkgFmri objects, newest
versions first, for packages matching those found in the
'versions' list, and a dict of unmatched versions indexed by
match criteria.
'versions' should be a list of strings of the format:
release,build_release-branch:datetime
...with a value of '*' provided for any component to be ignored.
'*' or '?' may be used within each component value and will act
as wildcard characters ('*' for one or more characters, '?' for
a single character).
"""
try:
c = self._depot.repo.catalog
except srepo.RepositoryMirrorError:
return tuple(), {}
return pkg.catalog.extract_matching_fmris(c.fmris(),
versions=versions)
def info(self, fmri_strings, info_needed, excludes=misc.EmptyI):
"""Gathers information about fmris. fmri_strings is a list
of fmri_names for which information is desired. It
returns a dictionary of lists. The keys for the dictionary are
the constants specified in the class definition. The values are
lists of PackageInfo objects or strings."""
bad_opts = info_needed - PackageInfo.ALL_OPTIONS
if bad_opts:
raise api_errors.UnrecognizedOptionsToInfo(bad_opts)
fmris = []
notfound = []
illegals = []
for pattern in fmri_strings:
try:
pfmri = None
pfmri = self.get_matching_pattern_fmris(pattern)
except pkg.fmri.IllegalFmri, e:
illegals.append(pattern)
continue
else:
fmris.extend(pfmri[0])
if not pfmri:
notfound.append(pattern)
repo_cat = self._depot.repo.catalog
# Set of options that can use catalog data.
cat_opts = frozenset([PackageInfo.SUMMARY,
PackageInfo.CATEGORIES, PackageInfo.DESCRIPTION,
PackageInfo.DEPENDENCIES])
# Set of options that require manifest retrieval.
act_opts = PackageInfo.ACTION_OPTIONS - \
frozenset([PackageInfo.DEPENDENCIES])
pis = []
for f in fmris:
pub = name = version = release = None
build_release = branch = packaging_date = None
if PackageInfo.IDENTITY in info_needed:
pub, name, version = f.tuple()
pub = pkg.fmri.strip_pub_pfx(pub)
release = version.release
build_release = version.build_release
branch = version.branch
packaging_date = \
version.get_timestamp().strftime("%c")
states = None
links = hardlinks = files = dirs = dependencies = None
summary = size = licenses = cat_info = description = \
None
if cat_opts & info_needed:
summary, description, cat_info, dependencies = \
_get_pkg_cat_data(repo_cat, info_needed,
excludes=excludes, pfmri=f)
if cat_info is not None:
cat_info = [
PackageCategory(scheme, cat)
for scheme, cat in cat_info
]
if (frozenset([PackageInfo.SIZE,
PackageInfo.LICENSES]) | act_opts) & info_needed:
mfst = manifest.Manifest()
mfst.set_fmri(None, f)
try:
mpath = os.path.join(
self._depot.repo.manifest_root,
f.get_dir_path())
except pkg.fmri.FmriError, e:
notfound.append(f)
continue
if not os.path.exists(mpath):
notfound.append(f)
continue
mfst.set_content(file(mpath).read())
if PackageInfo.LICENSES in info_needed:
licenses = self.__licenses(mfst)
if PackageInfo.SIZE in info_needed:
size = mfst.get_size(excludes=excludes)
if act_opts & info_needed:
if PackageInfo.LINKS in info_needed:
links = list(
mfst.gen_key_attribute_value_by_type(
"link", excludes))
if PackageInfo.HARDLINKS in info_needed:
hardlinks = list(
mfst.gen_key_attribute_value_by_type(
"hardlink", excludes))
if PackageInfo.FILES in info_needed:
files = list(
mfst.gen_key_attribute_value_by_type(
"file", excludes))
if PackageInfo.DIRS in info_needed:
dirs = list(
mfst.gen_key_attribute_value_by_type(
"dir", excludes))
pis.append(PackageInfo(pkg_stem=name, summary=summary,
category_info_list=cat_info, states=states,
publisher=pub, version=release,
build_release=build_release, branch=branch,
packaging_date=packaging_date, size=size,
pfmri=str(f), licenses=licenses, links=links,
hardlinks=hardlinks, files=files, dirs=dirs,
dependencies=dependencies, description=description))
def main_func():
global_settings.client_name = "pkgsign"
try:
opts, pargs = getopt.getopt(sys.argv[1:], "a:c:i:k:ns:D:",
["help", "no-index", "no-catalog"])
except getopt.GetoptError as e:
usage(_("illegal global option -- {0}").format(e.opt))
show_usage = False
sig_alg = "rsa-sha256"
cert_path = None
key_path = None
chain_certs = []
add_to_catalog = True
set_alg = False
dry_run = False
repo_uri = os.getenv("PKG_REPO", None)
for opt, arg in opts:
if opt == "-a":
sig_alg = arg
set_alg = True
elif opt == "-c":
cert_path = os.path.abspath(arg)
if not os.path.isfile(cert_path):
usage(_("{0} was expected to be a certificate "
"but isn't a file.").format(cert_path))
elif opt == "-i":
p = os.path.abspath(arg)
if not os.path.isfile(p):
usage(_("{0} was expected to be a certificate "
"but isn't a file.").format(p))
chain_certs.append(p)
elif opt == "-k":
key_path = os.path.abspath(arg)
if not os.path.isfile(key_path):
usage(_("{0} was expected to be a key file "
"but isn't a file.").format(key_path))
elif opt == "-n":
dry_run = True
elif opt == "-s":
repo_uri = misc.parse_uri(arg)
elif opt == "--help":
show_usage = True
elif opt == "--no-catalog":
add_to_catalog = False
elif opt == "-D":
try:
key, value = arg.split("=", 1)
DebugValues.set_value(key, value)
except (AttributeError, ValueError):
error(_("{opt} takes argument of form "
"name=value, not {arg}").format(
opt=opt, arg=arg))
if show_usage:
usage(retcode=EXIT_OK)
if not repo_uri:
usage(_("a repository must be provided"))
if key_path and not cert_path:
usage(_("If a key is given to sign with, its associated "
"certificate must be given."))
if cert_path and not key_path:
usage(_("If a certificate is given, its associated key must be "
"given."))
if chain_certs and not cert_path:
usage(_("Intermediate certificates are only valid if a key "
"and certificate are also provided."))
if not pargs:
usage(_("At least one fmri or pattern must be provided to "
"sign."))
if not set_alg and not key_path:
sig_alg = "sha256"
s, h = actions.signature.SignatureAction.decompose_sig_alg(sig_alg)
if h is None:
usage(_("{0} is not a recognized signature algorithm.").format(
sig_alg))
if s and not key_path:
usage(_("Using {0} as the signature algorithm requires that a "
"key and certificate pair be presented using the -k and -c "
"options.").format(sig_alg))
if not s and key_path:
usage(_("The {0} hash algorithm does not use a key or "
"certificate. Do not use the -k or -c options with this "
"algorithm.").format(sig_alg))
if DebugValues:
reload(digest)
errors = []
t = misc.config_temp_root()
temp_root = tempfile.mkdtemp(dir=t)
del t
cache_dir = tempfile.mkdtemp(dir=temp_root)
incoming_dir = tempfile.mkdtemp(dir=temp_root)
chash_dir = tempfile.mkdtemp(dir=temp_root)
cert_dir = tempfile.mkdtemp(dir=temp_root)
try:
chain_certs = [
__make_tmp_cert(cert_dir, c) for c in chain_certs
]
if cert_path is not None:
cert_path = __make_tmp_cert(cert_dir, cert_path)
xport, xport_cfg = transport.setup_transport()
xport_cfg.add_cache(cache_dir, readonly=False)
xport_cfg.incoming_root = incoming_dir
# Configure publisher(s)
transport.setup_publisher(repo_uri, "source", xport,
xport_cfg, remote_prefix=True)
pats = pargs
successful_publish = False
concrete_fmris = []
unmatched_pats = set(pats)
all_pats = frozenset(pats)
get_all_pubs = False
pub_prefs = set()
# Gather the publishers whose catalogs will be needed.
for pat in pats:
try:
p_obj = fmri.MatchingPkgFmri(pat)
except fmri.IllegalMatchingFmri as e:
errors.append(e)
continue
pub_prefix = p_obj.get_publisher()
if pub_prefix:
pub_prefs.add(pub_prefix)
else:
get_all_pubs = True
# Check each publisher for matches to our patterns.
for p in xport_cfg.gen_publishers():
if not get_all_pubs and p.prefix not in pub_prefs:
continue
cat = fetch_catalog(p, xport, temp_root)
ms, tmp1, u = cat.get_matching_fmris(pats)
# Find which patterns matched.
matched_pats = all_pats - u
# Remove those patterns from the unmatched set.
unmatched_pats -= matched_pats
for v_list in ms.values():
concrete_fmris.extend([(v, p) for v in v_list])
if unmatched_pats:
raise api_errors.PackageMatchErrors(
unmatched_fmris=unmatched_pats)
for pfmri, src_pub in sorted(set(concrete_fmris)):
try:
# Get the existing manifest for the package to
# be signed.
m_str = xport.get_manifest(pfmri,
content_only=True, pub=src_pub)
m = manifest.Manifest()
m.set_content(content=m_str)
# Construct the base signature action.
attrs = { "algorithm": sig_alg }
a = actions.signature.SignatureAction(cert_path,
**attrs)
a.hash = cert_path
# Add the action to the manifest to be signed
# since the action signs itself.
m.add_action(a, misc.EmptyI)
# Set the signature value and certificate
# information for the signature action.
a.set_signature(m.gen_actions(),
key_path=key_path, chain_paths=chain_certs,
chash_dir=chash_dir)
# The hash of 'a' is currently a path, we need
# to find the hash of that file to allow
# comparison to existing signatures.
hsh = None
if cert_path:
# Action identity still uses the 'hash'
# member of the action, so we need to
# stay with the sha1 hash.
hsh, _dummy = \
misc.get_data_digest(cert_path,
hash_func=hashlib.sha1)
# Check whether the signature about to be added
# is identical, or almost identical, to existing
# signatures on the package. Because 'a' has
# already been added to the manifest, it is
# generated by gen_actions_by_type, so the cnt
# must be 2 or higher to be an issue.
cnt = 0
almost_identical = False
for a2 in m.gen_actions_by_type("signature"):
try:
if a.identical(a2, hsh):
cnt += 1
except api_errors.AlmostIdentical as e:
e.pkg = pfmri
errors.append(e)
almost_identical = True
if almost_identical:
continue
if cnt == 2:
continue
elif cnt > 2:
raise api_errors.DuplicateSignaturesAlreadyExist(pfmri)
assert cnt == 1, "Cnt was:{0}".format(cnt)
if not dry_run:
# Append the finished signature action
# to the published manifest.
t = trans.Transaction(repo_uri,
pkg_name=str(pfmri), xport=xport,
pub=src_pub)
try:
t.append()
t.add(a)
for c in chain_certs:
t.add_file(c)
t.close(add_to_catalog=
add_to_catalog)
except:
if t.trans_id:
t.close(abandon=True)
raise
msg(_("Signed {0}").format(pfmri.get_fmri(
include_build=False)))
successful_publish = True
except (api_errors.ApiException, fmri.FmriError,
trans.TransactionError) as e:
errors.append(e)
if errors:
error("\n".join([str(e) for e in errors]))
if successful_publish:
return EXIT_PARTIAL
else:
return EXIT_OOPS
return EXIT_OK
except api_errors.ApiException as e:
error(e)
return EXIT_OOPS
finally:
shutil.rmtree(temp_root)
def __do_alter_verify(self,
pfmri,
verbose=False,
quiet=False,
exit=0,
parsable=False):
# Alter the owner, group, mode, and timestamp of all files (and
# directories) to something different than the package declares.
m = manifest.Manifest()
m.set_content(self.get_img_manifest(pfmri))
ctime = time.time() - 1000
for a in m.gen_actions():
if a.name not in ("file", "dir"):
# Only want file or dir actions.
continue
ubin = portable.get_user_by_name("bin", None, False)
groot = portable.get_group_by_name("root", None, False)
fname = a.attrs["path"]
fpath = os.path.join(self.get_img_path(), fname)
os.chown(fpath, ubin, groot)
os.chmod(fpath, misc.PKG_RO_FILE_MODE)
os.utime(fpath, (ctime, ctime))
# Call pkg fix to fix them.
fix_cmd = "fix"
if verbose:
fix_cmd += " -v"
if quiet:
fix_cmd += " -q"
if parsable:
fix_cmd += " --parsable=0"
self.pkg("{0} {1}".format(fix_cmd, pfmri), exit=exit)
if exit != 0:
return exit
editables = []
# Now verify that fix actually fixed them.
for a in m.gen_actions():
if a.name not in ("file", "dir"):
# Only want file or dir actions.
continue
# Validate standard attributes.
self.validate_fsobj_attrs(a)
# Now validate attributes that require special handling.
fname = a.attrs["path"]
fpath = os.path.join(self.get_img_path(), fname)
lstat = os.lstat(fpath)
# Verify that preserved files don't get renamed, and
# the new ones are not installed if the file wasn't
# missing already.
preserve = a.attrs.get("preserve")
if preserve == "renamenew":
self.assert_(not os.path.exists(fpath + ".new"))
elif preserve == "renameold":
self.assert_(not os.path.exists(fpath + ".old"))
if preserve:
editables.append("{0}".format(a.attrs["path"]))
# Verify timestamp (if applicable).
ts = a.attrs.get("timestamp")
if ts:
expected = misc.timestamp_to_time(ts)
actual = lstat.st_mtime
if preserve:
self.assertNotEqual(
expected, actual, "timestamp expected {expected} == "
"actual {actual} for "
"{fname}".format(expected=expected,
actual=actual,
fname=fname))
else:
self.assertEqual(
expected, actual, "timestamp expected {expected} != "
"actual {actual} for "
"{fname}".format(expected=expected,
actual=actual,
fname=fname))
# Verify the parsable output (if applicable).
if parsable:
if editables:
self.assertEqualParsable(
self.output,
affect_packages=["{0}".format(pfmri)],
change_editables=[["updated", editables]])
else:
self.assertEqualParsable(self.output,
affect_packages=["{0}".format(pfmri)])
def addmanifest(self, root, mfile, arch, modechecks, exceptions):
"""Treats the specified input file as a pkg(7) package
manifest, and extends the ManifestTree dictionary with entries
for the actions therein.
"""
mfest = manifest.Manifest()
try:
mfest.set_content(open(os.path.join(root, mfile)).read())
except IOError as exc:
raise IOError("cannot read manifest: %s" % str(exc))
except actions.ActionError as exc:
raise ManifestParsingError(mfile, str(exc))
#
# Make sure the manifest is applicable to the user-specified
# architecture. Assumption: if variant.arch is not an
# attribute of the manifest, then the package should be
# installed on all architectures.
#
if arch not in mfest.attributes.get("variant.arch", (arch, )):
return
modewarnings = set()
for action in mfest.gen_actions():
if "path" not in action.attrs or \
not ActionInfo.supported(action.name):
continue
#
# The dir action is currently fully specified, in that it
# lists owner, group, and mode attributes. If that
# changes in pkg(7) code, we'll need to revisit either this
# code or the ActionInfo() constructor. It's possible
# that the pkg(7) system could be extended to provide a
# mechanism for specifying directory permissions outside
# of the individual manifests that deliver files into
# those directories. Doing so at time of manifest
# processing would mean that validate_pkg continues to work,
# but doing so at time of publication would require updates.
#
#
# See pkgsend(1) for the use of NOHASH for objects with
# datastreams. Currently, that means "files," but this
# should work for any other such actions.
#
if getattr(action, "hash", "NOHASH") != "NOHASH":
path = action.hash
else:
path = action.attrs["path"]
#
# This is the wrong tool in which to enforce consistency
# on a set of manifests. So instead of comparing the
# different actions with the same "path" attribute, we
# use the first one.
#
if path in self:
continue
#
# As with the manifest itself, if an action has specified
# variant.arch, we look for the target architecture
# therein.
#
var = None
#
# The name of this method changed in pkg(7) build 150, we need to
# work with both sets.
#
if hasattr(action, 'get_variants'):
var = action.get_variants()
else:
var = action.get_variant_template()
if "variant.arch" in var and arch not in var["variant.arch"]:
return
self[path] = ActionInfo(action)
if modechecks is not None and path not in exceptions:
modewarnings.update(self[path].checkmodes(modechecks))
if len(modewarnings) > 0:
print("warning: unsafe permissions in %s" % mfile)
for w in sorted(modewarnings):
print(w)
print("")
def merge_fmris(server_list, fmri_list, variant_list, variant, basedir,
basename, get_files):
manifest_list = [
get_manifest(s, f) for s, f in zip(server_list, fmri_list)
]
# remove variant tags and package variant metadata
# from manifests since we're reassigning...
# this allows merging pre-tagged packages
for m in manifest_list:
for i, a in enumerate(m.actions[:]):
if variant in a.attrs:
del a.attrs[variant]
if a.name == "set" and a.attrs["name"] == variant:
del m.actions[i]
action_lists = manifest.Manifest.comm(*tuple(manifest_list))
# set fmri actions require special merge logic.
set_fmris = []
for l in action_lists:
for i, a in enumerate(l):
if not (a.name == "set" and a.attrs["name"] == "pkg.fmri"):
continue
set_fmris.append(a)
del l[i]
# If set fmris are present, then only the most recent one
# and add it back to the last action list.
if set_fmris:
def order(a, b):
f1 = pkg.fmri.PkgFmri(a.attrs["value"], "5.11")
f2 = pkg.fmri.PkgFmri(b.attrs["value"], "5.11")
return cmp(f1, f2)
set_fmris.sort(cmp=order)
action_lists[-1].insert(0, set_fmris[-1])
for a_list, v in zip(action_lists[0:-1], variant_list):
for a in a_list:
a.attrs[variant] = v
# combine actions into single list
allactions = reduce(lambda a, b: a + b, action_lists)
# figure out which variants are actually there for this pkg
actual_variant_list = [
v for m, v in zip(manifest_list, variant_list) if m != null_manifest
]
print "Merging %s for %s" % (basename, actual_variant_list)
# add set action to document which variants are supported
allactions.append(
actions.fromstr(
"set name=%s %s" %
(variant, " ".join(["value=%s" % a
for a in actual_variant_list]))))
allactions.sort()
m = manifest.Manifest()
m.set_content(content=allactions)
# urlquote to avoid problems w/ fmris w/ '/' character in name
basedir = os.path.join(basedir, urllib.quote(basename, ""))
if not os.path.exists(basedir):
os.makedirs(basedir)
m_path = os.path.join(basedir, "manifest")
m.store(m_path)
for f in fmri_list:
if f:
fmri = str(f).rsplit(":", 1)[0]
break
f_file = file(os.path.join(basedir, "fmri"), "w")
f_file.write(fmri)
f_file.close()
if get_files:
# generate list of hashes for each server; last is commom
already_seen = {}
def repeated(a, d):
if a in d:
return True
d[a] = 1
return False
action_sets = [
set(
[
a
for a in action_list
if hasattr(a, "hash") and not \
repeated(a.hash, already_seen)
]
)
for action_list in action_lists
]
# remove duplicate files (save time)
for server, action_set in zip(server_list + [server_list[0]],
action_sets):
if len(action_set) > 0:
fetch_files_byaction(server, action_set, basedir)
return 0
def test_info(self):
"""Testing information showed in /info/0."""
depot_url = self.dc.get_depot_url()
plist = self.pkgsend_bulk(depot_url, self.info20)
openurl = urlparse.urljoin(depot_url, "info/0/{0}".format(plist[0]))
content = urllib2.urlopen(openurl).read()
# Get text from content.
lines = content.splitlines()
info_dic = {}
attr_list = [
'Name', 'Summary', 'Publisher', 'Version', 'Build Release',
'Branch', 'Packaging Date', 'Size', 'Compressed Size', 'FMRI'
]
for line in lines:
fields = line.split(":", 1)
attr = fields[0].strip()
if attr == "License":
break
if attr in attr_list:
if len(fields) == 2:
info_dic[attr] = fields[1].strip()
else:
info_dic[attr] = ""
# Read manifest.
openurl = urlparse.urljoin(depot_url,
"manifest/0/{0}".format(plist[0]))
content = urllib2.urlopen(openurl).read()
manifest = man.Manifest()
manifest.set_content(content=content)
fmri_content = manifest.get("pkg.fmri", "")
# Check if FMRI is empty.
self.assert_(fmri_content)
pfmri = fmri.PkgFmri(fmri_content, None)
pub, name, ver = pfmri.tuple()
size, csize = manifest.get_size()
# Human version.
version = info_dic['Version']
hum_ver = ""
if '(' in version:
start = version.find('(')
end = version.rfind(')')
hum_ver = version[start + 1:end - len(version)]
version = version[:start - len(version)]
info_dic['Version'] = version.strip()
# Compare each attribute.
self.assertEqual(
info_dic["Summary"],
manifest.get("pkg.summary", manifest.get("description", "")))
self.assertEqual(info_dic["Version"], str(ver.release))
self.assertEqual(hum_ver, manifest.get("pkg.human-version", ""))
self.assertEqual(info_dic["Name"], name)
self.assertEqual(info_dic["Publisher"], pub)
self.assertEqual(info_dic["Build Release"], str(ver.build_release))
timestamp = datetime.datetime.strptime(info_dic["Packaging Date"],
"%a %b %d %H:%M:%S %Y")
self.assertEqual(timestamp, ver.get_timestamp())
self.assertEqual(info_dic["Size"], misc.bytes_to_str(size))
self.assertEqual(info_dic["Compressed Size"], misc.bytes_to_str(csize))
self.assertEqual(info_dic["FMRI"], fmri_content)
