def test_verify_parsable_output(self):
"""Test parsable output."""
self.image_create(self.rurl)
self.pkg("install foo")
# Test invalid option combo.
self.pkg_verify("-v --parsable 0 foo", exit=2)
self.pkg_verify("-H --parsable 0 foo", exit=2)
# Test invalid option value.
self.pkg_verify("--parsable 1 foo", exit=2)
self.pkg_verify("--parsable 0 foo")
out_json = json.loads(self.output)
self.assertTrue("pkg://test/[email protected],5.11-0:20160229T095441Z" in
out_json["item-messages"])
fmri_entry = out_json["item-messages"][
"pkg://test/[email protected],5.11-0:20160229T095441Z"]
self.assertTrue(fmri_entry["messages"][0]["msg_level"] == "info")
self.assertTrue("file: usr/bin/bobcat" in fmri_entry)
# Verify should fail if file is missing.
fpath = os.path.join(self.get_img_path(), "usr", "bin", "bobcat")
portable.remove(fpath)
self.pkg_verify("--parsable 0 foo", exit=1)
out_json = json.loads(self.output)
self.assertTrue("pkg://test/[email protected],5.11-0:20160229T095441Z" in
out_json["item-messages"])
fmri_entry = out_json["item-messages"][
"pkg://test/[email protected],5.11-0:20160229T095441Z"]
self.assertTrue(fmri_entry["messages"][0]["msg_type"] == "general")
self.assertTrue(fmri_entry["messages"][0]["msg_level"] == "error")
self.assertTrue("file: usr/bin/bobcat" in fmri_entry)
self.assertTrue(
fmri_entry["file: usr/bin/bobcat"][0]["msg_type"] == "general")
self.assertTrue(
fmri_entry["file: usr/bin/bobcat"][0]["msg_level"] == "error")
def __handle_error(self, out_json):
try:
if isinstance(out_json, six.string_types):
out_json = json.loads(out_json)
except Exception as ex:
out_json = {
"status": ERROR,
"errors": [{
"reason":
"invalid JSON output: {0}".format(out_json)
}]
}
if out_json["status"] not in [SUCCEED, NO_OP, ACTUATOR_TIMEOUT]:
err_code = out_json["status"]
err_message = ""
if "errors" in out_json:
errs = []
for e in out_json["errors"]:
if "reason" in e:
errs.append(e["reason"])
elif "info" in e:
errs.append(e["info"])
err_message = "{0}".format("\n".join(errs))
raise PkgException(err_code, err_message)
return out_json
def __fork_pkg_cmd(self, subcommand, pargs_json=None, opts_json=None):
try:
args = ["/usr/share/lib/pkg/rad-invoke"]
# If not JSON formatted string, need conversion.
if pargs_json and not isinstance(pargs_json,
six.string_types):
pargs_json = json.dumps(pargs_json)
if opts_json and not isinstance(opts_json,
six.string_types):
opts_json = json.dumps(opts_json)
if self.__image_path:
args.extend(["-R", self.__image_path])
if pargs_json:
args.extend(["--pargs", pargs_json])
if opts_json:
args.extend(["--opts", opts_json])
args.extend(["--prog-delay",
str(self.__progress_interval)])
args.append(subcommand)
p = subprocess.Popen(args, env=os.environ,
stdout=subprocess.PIPE)
actualret = None
# Process output JSON lines.
while True:
out_line = p.stdout.readline()
if out_line == b'' and p.poll() is not None:
break
if out_line:
out_json = json.loads(out_line)
# This indicates it is progress output.
if "phase" in out_json:
self._prog_event_handler(
out_json)
# This indicates it is the actual
# return.
elif "status" in out_json:
actualret = out_json
if not actualret:
return {"status": ERROR, "errors": [{"reason":
"no result collected in fork mode."}]}
return actualret
except Exception as ex:
return {"status": ERROR, "errors": [{"reason": str(ex)}
]}
def parse(proxy_host, data):
"""Reads the pkg(5) publisher JSON formatted data at 'location'
or from the provided file-like object 'fileobj' and returns a
tuple. The first element of the tuple is a list of publisher objects.
The second element is a dictionary of image properties.
'proxy_host' is the string to replace the special string
'http://<sysrepo>' with when it starts any uri.
'data' is a string containing the p5s data.
"""
def transform_urls(urls):
res = []
for val in urls:
# If the URI contains <sysrepo> then it's served
# directly by the system-repository.
if val.startswith("http://{0}".format(publisher.SYSREPO_PROXY)):
scheme, netloc, path, params, query, fragment =\
urlparse(val)
r = publisher.RepositoryURI(
urlunparse(
(scheme, proxy_host, path, params, query, fragment)))
else:
# This URI needs to be proxied through the
# system-repository, so we assign it a special
# ProxyURI, which gets replaced by the actual
# URI of the system-repository in
# imageconfig.BlendedConfig.__merge_publishers
r = publisher.RepositoryURI(val)
r.proxies = [publisher.ProxyURI(None, system=True)]
res.append(r)
return res
try:
dump_struct = json.loads(data)
except ValueError as e:
# Not a valid JSON file.
raise api_errors.InvalidP5SFile(e)
try:
ver = int(dump_struct["version"])
except KeyError:
raise api_errors.InvalidP5SFile(_("missing version"))
except ValueError:
raise api_errors.InvalidP5SFile(_("invalid version"))
if ver > CURRENT_VERSION:
raise api_errors.UnsupportedP5SFile()
pubs = []
props = {}
try:
plist = dump_struct.get("publishers", [])
# For each set of publisher information in the parsed p5s file,
# build a Publisher object.
for p in plist:
alias = p.get("alias", None)
prefix = p.get("name", None)
sticky = p.get("sticky", True)
if not prefix:
prefix = "Unknown"
pub = publisher.Publisher(prefix, alias=alias, sticky=sticky)
v = p.get("signature-policy")
if v is not None:
pub.properties["signature-policy"] = v
v = p.get("signature-required-names")
if v is not None:
pub.properties["signature-required-names"] = v
r = p.get("repository", None)
if r:
rargs = {}
for prop in ("collection_type", "description", "name",
"refresh_seconds", "sticky"):
val = r.get(prop, None)
if val is None or val == "None":
continue
rargs[prop] = val
for prop in ("legal_uris", "related_uris"):
val = r.get(prop, [])
if not isinstance(val, list):
continue
rargs[prop] = val
for prop in ("mirrors", "origins"):
urls = r.get(prop, [])
if not isinstance(urls, list):
continue
rargs[prop] = transform_urls(urls)
repo = publisher.Repository(**rargs)
pub.repository = repo
pubs.append(pub)
props["publisher-search-order"] = \
dump_struct["image_properties"]["publisher-search-order"]
sig_pol = dump_struct["image_properties"].get("signature-policy")
if sig_pol is not None:
props["signature-policy"] = sig_pol
req_names = dump_struct["image_properties"].get(
"signature-required-names")
if req_names is not None:
props["signature-required-names"] = req_names
except (api_errors.PublisherError, TypeError, ValueError) as e:
raise api_errors.InvalidP5SFile(str(e))
return pubs, props
def test_01_basics(self):
"""Basic fix test: install the amber package, modify one of the
files, and make sure it gets fixed. """
self.image_create(self.rurl)
self.pkg("install [email protected]")
# Test invalid option combo.
self.pkg("fix -v --parsable 0 foo", exit=2)
self.pkg("fix -H --parsable 0 foo", exit=2)
# Test invalid option value.
self.pkg("fix --parsable 1 foo", exit=2)
# Test parsable output.
self.pkg("fix --parsable 0 amber", exit=4)
out_json = json.loads(self.output)
item_id = list(out_json["item-messages"].keys())[0]
self.assertTrue(out_json["item-messages"][item_id]["messages"][0]
["msg_level"] == "info")
# Test unpackaged option.
self.pkg("fix --unpackaged", exit=4)
self.pkg("fix --parsable 0 --unpackaged", exit=4)
out_json = json.loads(self.output)
subitem_id = list(out_json["item-messages"]["unpackaged"].keys())[0]
self.assertTrue(out_json["item-messages"]["unpackaged"][subitem_id][0]
["msg_level"] == "info")
index_dir = self.get_img_api_obj().img.index_dir
index_file = os.path.join(index_dir, "main_dict.ascii.v2")
orig_mtime = os.stat(index_file).st_mtime
time.sleep(1)
victim = "etc/amber2"
# Initial size
size1 = self.file_size(victim)
# Corrupt the file
self.file_append(victim, "foobar")
# Make sure the size actually changed
size2 = self.file_size(victim)
self.assertNotEqual(size1, size2)
# Verify that unprivileged users are handled by fix.
self.pkg("fix amber", exit=1, su_wrap=True)
self.pkg("fix --unpackaged -nv amber")
self.assertTrue("----" in self.output and "UNPACKAGED" in self.output)
# Fix the package
self.pkg("fix amber")
# Make sure it's the same size as the original
size2 = self.file_size(victim)
self.assertEqual(size1, size2)
# check that we didn't reindex
new_mtime = os.stat(index_file).st_mtime
self.assertEqual(orig_mtime, new_mtime)
# Verify that removing the publisher of a package that needs
# fixing results in graceful failure (not a traceback).
self.file_append(victim, "foobar")
self.pkg("set-publisher -P --no-refresh -g {0} foo".format(self.rurl))
self.pkg("unset-publisher test")
self.pkg("fix", exit=1)
def parse(data=None, fileobj=None, location=None):
"""Reads the pkg(5) publisher JSON formatted data at 'location'
or from the provided file-like object 'fileobj' and returns a
list of tuples of the format (publisher object, pkg_names).
pkg_names is a list of strings representing package names or
FMRIs. If any pkg_names not specific to a publisher were
provided, the last tuple returned will be of the format (None,
pkg_names).
'data' is an optional string containing the p5i data.
'fileobj' is an optional file-like object that must support a
'read' method for retrieving data.
'location' is an optional string value that should either start
with a leading slash and be pathname of a file or a URI string.
If it is a URI string, supported protocol schemes are 'file',
'ftp', 'http', and 'https'.
'data' or 'fileobj' or 'location' must be provided."""
if data is None and location is None and fileobj is None:
raise api_errors.InvalidResourceLocation(location)
if location is not None:
if location.find("://") == -1 and \
not location.startswith("file:/"):
# Convert the file path to a URI.
location = os.path.abspath(location)
location = urlunparse(
("file", "", pathname2url(location), "", "", ""))
try:
fileobj = urlopen(location)
except (EnvironmentError, ValueError, HTTPError) as e:
raise api_errors.RetrievalError(e, location=location)
try:
if data is not None:
dump_struct = json.loads(data)
else:
dump_struct = json.load(fileobj)
except (EnvironmentError, HTTPError) as e:
raise api_errors.RetrievalError(e)
except ValueError as e:
# Not a valid JSON file.
raise api_errors.InvalidP5IFile(e)
try:
ver = int(dump_struct["version"])
except KeyError:
raise api_errors.InvalidP5IFile(_("missing version"))
except ValueError:
raise api_errors.InvalidP5IFile(_("invalid version"))
if ver > CURRENT_VERSION:
raise api_errors.UnsupportedP5IFile()
result = []
try:
plist = dump_struct.get("publishers", [])
for p in plist:
alias = p.get("alias", None)
prefix = p.get("name", None)
if not prefix:
prefix = "Unknown"
pub = publisher.Publisher(prefix, alias=alias)
pkglist = p.get("packages", [])
result.append((pub, pkglist))
for r in p.get("repositories", []):
rargs = {}
for prop in ("collection_type", "description", "name",
"refresh_seconds", "registration_uri"):
val = r.get(prop, None)
if val is None or val == "None":
continue
rargs[prop] = val
for prop in ("legal_uris", "mirrors", "origins",
"related_uris"):
val = r.get(prop, [])
if not isinstance(val, list):
continue
rargs[prop] = val
repo = publisher.Repository(**rargs)
pub.repository = repo
pkglist = dump_struct.get("packages", [])
if pkglist:
result.append((None, pkglist))
except (api_errors.PublisherError, TypeError, ValueError) as e:
raise api_errors.InvalidP5IFile(str(e))
return result
def test_unpackaged(self):
"""Test unpackaged option."""
self.image_create(self.rurl)
self.pkg("install foo")
self.pkg_verify("--unpackaged")
self.assertTrue("ERROR" not in self.output and
"WARNING" not in self.output and "Unpackaged" in
self.output and "UNPACKAGED" in self.output)
self.assertTrue("----" not in self.output)
# Test verbose.
self.pkg_verify("-v --unpackaged")
self.assertTrue("----" in self.output)
self.assertTrue("ERROR" not in self.output and
"WARNING" not in self.output and "Unpackaged" in
self.output and "UNPACKAGED" in self.output)
self.assertTrue("ERROR" not in self.output and
"WARNING" not in self.output and "bobcat" in
self.output and "UNPACKAGED" in self.output)
# Test omit header.
self.pkg_verify("--unpackaged -H")
self.assertTrue("----" not in self.output)
self.assertTrue("ERROR" not in self.output and
"WARNING" not in self.output and "Unpackaged" in
self.output and "UNPACKAGED" not in self.output)
# Test unpackaged only.
self.pkg_verify("--unpackaged-only -v")
self.assertTrue("----" not in self.output)
self.assertTrue("ERROR" not in self.output and
"WARNING" not in self.output and "Unpackaged" in
self.output and "UNPACKAGED" in self.output)
self.assertTrue("ERROR" not in self.output and
"WARNING" not in self.output and "Unpackaged" in
self.output and "bobcat" not in self.output)
# Test quiet result.
self.pkg_verify("-q --unpackaged")
self.assertTrue(not self.output)
# Test invalid usage.
self.pkg_verify("--unpackaged-only foo", exit=2)
self.pkg_verify("--unpackaged --unpackaged-only", exit=2)
self.pkg_verify("-H --parsable 0 --unpackaged", exit=2)
# Test --unpackaged with package arguments.
self.pkg_verify("--unpackaged -v foo")
self.assertTrue("----" in self.output and "bobcat" in
self.output and "UNPACKAGED" in self.output)
self.pkg_verify("--parsable 0 --unpackaged foo")
out_json = json.loads(self.output)
self.assertTrue("pkg://test/[email protected],5.11-0:20160229T095441Z"
in out_json["item-messages"])
fmri_entry = out_json["item-messages"][
"pkg://test/[email protected],5.11-0:20160229T095441Z"]
self.assertTrue(fmri_entry["messages"][0]["msg_level"]
== "info")
self.assertTrue("file: usr/bin/bobcat" in fmri_entry)
# Test parsable output for --unpackaged.
self.pkg_verify("--parsable 0 --unpackaged")
out_json = json.loads(self.output)
self.assertTrue("unpackaged" in out_json["item-messages"]
and out_json["item-messages"]["unpackaged"])
out_entry = out_json["item-messages"]["unpackaged"]
self.assertTrue("dir" in list(out_entry.keys())[0] or "file" in
out_entry.keys()[0])
self.assertTrue(out_entry[list(out_entry.keys())[0]][0]["msg_level"]
== "info")
self.assertTrue(out_entry[list(out_entry.keys())[0]][0]["msg_type"]
== "unpackaged")
self.assertTrue("pkg://test/[email protected],5.11-0:20160229T095441Z"
in out_json["item-messages"])
fmri_entry = out_json["item-messages"][
"pkg://test/[email protected],5.11-0:20160229T095441Z"]
self.assertTrue(fmri_entry["messages"][0]["msg_level"]
== "info")
self.assertTrue("file: usr/bin/bobcat" in fmri_entry)
# Test parsable output for --unpackaged-only.
self.pkg_verify("--parsable 0 --unpackaged-only")
out_json = json.loads(self.output)
self.assertTrue("unpackaged" in out_json["item-messages"]
and out_json["item-messages"]["unpackaged"])
out_entry = out_json["item-messages"]["unpackaged"]
self.assertTrue("dir" in list(out_entry.keys())[0] or "file" in
out_entry.keys()[0])
self.assertTrue(out_entry[list(out_entry.keys())[0]][0]["msg_level"]
== "info")
self.assertTrue(out_entry[list(out_entry.keys())[0]][0]["msg_type"]
== "unpackaged")
self.assertTrue("pkg://test/[email protected],5.11-0:20160229T095441Z"
not in out_json["item-messages"])
