def test_action_license(self):
"""Test license action attributes."""
# Verify license attributes for must-accept / must-display
# contain expected values.
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" custom="foo" '
'bool_val=true')
self.assertEqual(a.must_accept, False)
self.assertEqual(a.must_display, False)
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" must-accept=true '
'must-display=False')
self.assertEqual(a.must_accept, True)
self.assertEqual(a.must_display, False)
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" must-accept=True '
'must-display=true')
self.assertEqual(a.must_accept, True)
self.assertEqual(a.must_display, True)
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" must-accept=True ')
self.assertEqual(a.must_accept, True)
self.assertEqual(a.must_display, False)
def get_directories(self, excludes):
""" return a list of directories implicitly or
explicitly referenced by this object"""
mpath = self.__file_path("manifest.dircache")
if not os.path.exists(mpath):
# no cached copy
if not self.loaded:
# need to load from disk
self.__load()
# generate actions that contain directories
alist = [
actions.fromstr(s.strip())
for s in self.__gen_dirs_to_str(
self.__actions_to_dirs())
]
else:
# we have cached copy on disk; use it
f = file(mpath)
alist = [actions.fromstr(s.strip()) for s in f]
f.close()
s = set([
a.attrs["path"]
for a in alist
if a.include_this(excludes)
])
return list(s)
def test_action_license(self):
"""Test license action attributes."""
# Verify license attributes for must-accept / must-display
# contain expected values.
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" custom="foo" '
'bool_val=true')
self.assertEqual(a.must_accept, False)
self.assertEqual(a.must_display, False)
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" must-accept=true '
'must-display=False')
self.assertEqual(a.must_accept, True)
self.assertEqual(a.must_display, False)
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" must-accept=True '
'must-display=true')
self.assertEqual(a.must_accept, True)
self.assertEqual(a.must_display, True)
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" must-accept=True ')
self.assertEqual(a.must_accept, True)
self.assertEqual(a.must_display, False)
def test_action_sig_str(self):
sig_act = action.fromstr("signature 54321 algorithm=baz")
for s in self.act_strings:
# action.sig_str should return an identical string each
# time it's called. Also, parsing the result of
# sig_str so produce the same action.
a = action.fromstr(s)
s2 = a.sig_str(sig_act, generic.Action.sig_version)
s3 = a.sig_str(sig_act, generic.Action.sig_version)
# If s2 is None, then s was a different signature
# action, so there is no output to parse.
if s2 is None:
continue
self.assertEqual(s2, s3)
a2 = action.fromstr(s2)
if a.different(a2):
self.debug("a1 " + str(a))
self.debug("a2 " + str(a2))
self.assertTrue(not a.different(a2))
s4 = a.sig_str(sig_act, generic.Action.sig_version)
self.assertEqual(s2, s4)
# Test that using an unknown sig_version triggers the
# appropriate exception.
self.assertRaises(api_errors.UnsupportedSignatureVersion,
sig_act.sig_str, sig_act, -1)
a = action.fromstr(self.act_strings[0])
self.assertRaises(api_errors.UnsupportedSignatureVersion, a.sig_str,
sig_act, -1)
# Test that the sig_str of a signature action other than the
# argument action is None.
sig_act2 = action.fromstr("signature 98765 algorithm=foobar")
self.assertTrue(
sig_act.sig_str(sig_act2, generic.Action.sig_version) is None)
self.assertTrue(
sig_act2.sig_str(sig_act, generic.Action.sig_version) is None)
def assertMalformed(self, text):
malformed = False
try:
action.fromstr(text)
except action.MalformedActionError, e:
assert e.actionstr == text
malformed = True
def assertInvalid(self, text):
invalid = False
try:
action.fromstr(text)
except action.InvalidActionError:
invalid = True
# If the action isn't invalid, something is wrong.
self.assert_(invalid, "Action not invalid: " + text)
def assertInvalid(self, text):
invalid = False
try:
action.fromstr(text)
except action.InvalidActionError:
invalid = True
# If the action isn't invalid, something is wrong.
self.assertTrue(invalid, "Action not invalid: " + text)
def assertMalformed(self, text):
malformed = False
try:
action.fromstr(text)
except action.MalformedActionError as e:
assert e.actionstr == text
malformed = True
# If the action isn't malformed, something is wrong.
self.assert_(malformed, "Action not malformed: " + text)
def adjust_dep_action(pkg, act, latest_ref_pkgs, reversioned_pkgs, ref_xport):
"""Adjust dependency FMRIs of action 'act' if it is of type depend.
The adjusted action will reference only FMRIs which are present in the
reversioned repo. """
modified = False
# Drop signatures (changed dependency will void signature value).
if act.name == "signature":
return
# Ignore anything other than depend actions.
elif act.name != "depend":
return act
# Require-any deps are list so convert every dep FMRI into a list.
fmris = act.attrlist("fmri")
new_dep = []
for f in fmris:
new_f = get_dep_fmri_str(f, pkg, act, latest_ref_pkgs,
reversioned_pkgs, ref_xport)
if not modified and f != new_f:
modified = True
new_dep.append(new_f)
if not modified:
return act
if len(new_dep) == 1:
new_dep = new_dep[0]
nact = actions.fromstr(str(act))
nact.attrs["fmri"] = new_dep
return nact
def __run_search(paths, api_inst):
"""Function which interfaces with the search engine and extracts the
fmri and variants from the actions which deliver the paths being
searched for.
'paths' is the paths to search for.
'api_inst' is an ImageInterface which references the current image."""
qs = [
api.Query(p, case_sensitive=False, return_actions=True) for p in paths
]
search_res = api_inst.local_search(qs)
res = []
try:
for num, pub, (version, return_type, (pfmri, match, a_str)) \
in search_res:
pfmri = fmri.PkgFmri(pfmri)
m = api_inst.img.get_manifest(pfmri)
vars = variants.VariantSets(
actions.fromstr(a_str.rstrip()).get_variants())
vars.merge_unknown(m.get_all_variants())
res.append((pfmri, vars))
except api_errors.SlowSearchUsed:
pass
return res
def gen_actions_by_type(self, atype, excludes=EmptyI):
""" generate actions of the specified type;
use already in-memory stuff if already loaded,
otherwise use per-action types files"""
if self.loaded: #if already loaded, use in-memory cached version
# invoke subclass method to generate action by action
for a in Manifest.gen_actions_by_type(self, atype, excludes):
yield a
return
mpath = self.__file_path("manifest.dircache")
if not os.path.exists(mpath):
# no cached copy :-(
if not self.loaded:
# get manifest from disk
self.__load()
# invoke subclass method to generate action by action
for a in Manifest.gen_actions_by_type(self, atype, excludes):
yield a
else:
# we have a cached copy - use it
mpath = self.__file_path("manifest.%s" % atype)
if not os.path.exists(mpath):
return # no such action in this manifest
f = file(mpath)
for l in f:
a = actions.fromstr(l.strip())
if a.include_this(excludes):
yield a
f.close()
def __content_to_actions(self, content):
accumulate = ""
lineno = 0
errors = []
for l in content.splitlines():
lineno += 1
l = l.lstrip()
if l.endswith("\\"): # allow continuation chars
accumulate += l[0:-1] # elide backslash
continue
elif accumulate:
l = accumulate + l
accumulate = ""
if not l or l[0] == "#": # ignore blank lines & comments
continue
try:
yield actions.fromstr(l)
except actions.ActionError, e:
# Accumulate errors and continue so that as
# much of the action data as possible can be
# parsed.
e.fmri = self.fmri
e.lineno = lineno
errors.append(e)
def __run_search(paths, api_inst):
"""Function which interfaces with the search engine and extracts the
fmri and variants from the actions which deliver the paths being
searched for.
'paths' is the paths to search for.
'api_inst' is an ImageInterface which references the current image."""
qs = [
api.Query(p, case_sensitive=False, return_actions=True)
for p in paths
]
search_res = api_inst.local_search(qs)
res = []
try:
for num, pub, (version, return_type, (pfmri, match, a_str)) \
in search_res:
pfmri = fmri.PkgFmri(pfmri)
m = api_inst.img.get_manifest(pfmri)
vars = variants.VariantSets(actions.fromstr(
a_str.rstrip()).get_variants())
vars.merge_unknown(m.get_all_variants())
res.append((pfmri, vars))
except api_errors.SlowSearchUsed:
pass
return res
def assertActionError(astr, error=action.InvalidActionAttributesError):
bad_act = action.fromstr(astr)
try:
bad_act.validate()
except Exception as e:
self.debug(str(e))
else:
self.debug("expected failure validating: {0}".format(astr))
self.assertRaises(error, bad_act.validate)
def __resolve_echo_line(l):
"""Given a line from a manifest, determines whether that line should
be repeated in the output file if echo manifest has been set."""
try:
act = actions.fromstr(l.rstrip())
except KeyboardInterrupt:
raise
except actions.ActionError:
return True
else:
return not act.name == "depend"
def __resolve_echo_line(l):
"""Given a line from a manifest, determines whether that line should
be repeated in the output file if echo manifest has been set."""
try:
act = actions.fromstr(l.rstrip())
except KeyboardInterrupt:
raise
except actions.ActionError:
return True
else:
return not act.name == "depend"
def __load_attributes(self):
"""Load attributes dictionary from cached set actions;
this speeds up pkg info a lot"""
mpath = self.__file_path("manifest.set")
if not os.path.exists(mpath):
return False
f = file(mpath)
for l in f:
a = actions.fromstr(l.strip())
if a.include_this(self.excludes):
self.fill_attributes(a)
f.close()
return True
def get_directories(self, excludes):
""" return a list of directories implicitly or
explicitly referenced by this object"""
mpath = self.__file_path("manifest.dircache")
if not os.path.exists(mpath):
# no cached copy
if not self.loaded:
# need to load from disk
self.__load()
# generate actions that contain directories
alist = [
actions.fromstr(s.strip())
for s in self.__gen_dirs_to_str(self.__actions_to_dirs())
]
else:
# we have cached copy on disk; use it
f = file(mpath)
alist = [actions.fromstr(s.strip()) for s in f]
f.close()
s = set([a.attrs["path"] for a in alist if a.include_this(excludes)])
return list(s)
def __load_attributes(self):
"""Load attributes dictionary from cached set actions;
this speeds up pkg info a lot"""
mpath = self.__file_path("manifest.set")
if not os.path.exists(mpath):
return False
f = file(mpath)
for l in f:
a = actions.fromstr(l.strip())
if a.include_this(self.excludes):
self.fill_attributes(a)
f.close()
return True
def test_action_sig_str(self):
sig_act = action.fromstr(
"signature 54321 algorithm=baz")
for s in self.act_strings:
# action.sig_str should return an identical string each
# time it's called. Also, parsing the result of
# sig_str so produce the same action.
a = action.fromstr(s)
s2 = a.sig_str(sig_act, generic.Action.sig_version)
s3 = a.sig_str(sig_act, generic.Action.sig_version)
# If s2 is None, then s was a different signature
# action, so there is no output to parse.
if s2 is None:
continue
self.assertEqual(s2, s3)
a2 = action.fromstr(s2)
if a.different(a2):
self.debug("a1 " + str(a))
self.debug("a2 " + str(a2))
self.assert_(not a.different(a2))
s4 = a.sig_str(sig_act, generic.Action.sig_version)
self.assertEqual(s2, s4)
# Test that using an unknown sig_version triggers the
# appropriate exception.
self.assertRaises(api_errors.UnsupportedSignatureVersion,
sig_act.sig_str, sig_act, -1)
a = action.fromstr(self.act_strings[0])
self.assertRaises(api_errors.UnsupportedSignatureVersion,
a.sig_str, sig_act, -1)
# Test that the sig_str of a signature action other than the
# argument action is None.
sig_act2 = action.fromstr(
"signature 98765 algorithm=foobar")
self.assert_(sig_act.sig_str(sig_act2,
generic.Action.sig_version) is None)
self.assert_(sig_act2.sig_str(sig_act,
generic.Action.sig_version) is None)
def __mangle_license(self, fmri):
repo = self.dc.get_repo()
m_path = repo.manifest(fmri)
with open(m_path, "r") as fh:
fmri_lines = fh.readlines()
with open(m_path, "w") as fh:
a = None
for l in fmri_lines:
if "license=copyright" in l:
continue
elif "path=etc/bronze1" in l:
a = actions.fromstr(l)
fh.write(l)
self.assertTrue(a)
l = """\
license {hash} license=foo chash={chash} pkg.csize={csize} \
pkg.size={size}""".format(hash=a.hash,
chash=a.attrs["chash"],
csize=a.attrs["pkg.csize"],
size=a.attrs["pkg.size"])
fh.write(l)
repo.rebuild()
def testMakedirs(self):
tmpdir = tempfile.mkdtemp()
# make the parent directory read-write
os.chmod(tmpdir, stat.S_IRWXU)
fpath = os.path.join(tmpdir, "f")
fopath = os.path.join(fpath, "o")
foopath = os.path.join(fopath, "o")
# make the leaf, and ONLY the leaf read-only
act = action.fromstr("dir path={0}".format(foopath))
act.makedirs(foopath, mode=stat.S_IREAD)
# Now make sure the directories leading up the leaf
# are read-write, and the leaf is readonly.
assert (os.stat(tmpdir).st_mode & (stat.S_IREAD | stat.S_IWRITE) != 0)
assert (os.stat(fpath).st_mode & (stat.S_IREAD | stat.S_IWRITE) != 0)
assert (os.stat(fopath).st_mode & (stat.S_IREAD | stat.S_IWRITE) != 0)
assert (os.stat(foopath).st_mode & stat.S_IREAD != 0)
assert (os.stat(foopath).st_mode & stat.S_IWRITE == 0)
# change it back to read/write so we can delete it
os.chmod(foopath, stat.S_IRWXU)
shutil.rmtree(tmpdir)
def testMakedirs(self):
tmpdir = tempfile.mkdtemp()
# make the parent directory read-write
os.chmod(tmpdir, stat.S_IRWXU)
fpath = os.path.join(tmpdir, "f")
fopath = os.path.join(fpath, "o")
foopath = os.path.join(fopath, "o")
# make the leaf, and ONLY the leaf read-only
act = action.fromstr("dir path=%s" % foopath)
act.makedirs(foopath, mode = stat.S_IREAD)
# Now make sure the directories leading up the leaf
# are read-write, and the leaf is readonly.
assert(os.stat(tmpdir).st_mode & (stat.S_IREAD | stat.S_IWRITE) != 0)
assert(os.stat(fpath).st_mode & (stat.S_IREAD | stat.S_IWRITE) != 0)
assert(os.stat(fopath).st_mode & (stat.S_IREAD | stat.S_IWRITE) != 0)
assert(os.stat(foopath).st_mode & stat.S_IREAD != 0)
assert(os.stat(foopath).st_mode & stat.S_IWRITE == 0)
# change it back to read/write so we can delete it
os.chmod(foopath, stat.S_IRWXU)
shutil.rmtree(tmpdir)
def test_action_tostr(self):
"""Test that actions convert to strings properly. This means
that we can feed the resulting string back into fromstr() and
get an identical action back."""
for s in self.act_strings:
self.debug(str(s))
a = action.fromstr(s)
s2 = str(a)
a2 = action.fromstr(s2)
if a.different(a2):
self.debug("a1 " + str(a))
self.debug("a2 " + str(a2))
self.assert_(not a.different(a2))
# The one place that invariant doesn't hold is when you specify
# the payload hash as the named attribute "hash", in which case
# the resulting re-stringification emits the payload hash as a
# positional attribute again ...
s = "file hash=abc123 path=usr/bin/foo mode=0755 owner=root group=bin"
self.debug(s)
a = action.fromstr(s)
s2 = str(a)
self.assert_(s2.startswith("file abc123 "))
self.assert_("hash=abc123" not in s2)
a2 = action.fromstr(s2)
self.assert_(not a.different(a2))
# ... unless of course the hash can't be represented that way.
d = {
"hash=abc=123": "abc=123",
"hash=\"one with spaces\"": "one with spaces",
"hash='one with \" character'": 'one with " character',
"hash=\"'= !@$%^\)(*\"": "'= !@$%^\)(*",
"""hash="\\"'= \\ " """: """"'= \\ """,
'\\': '\\'
}
astr = "file {0} path=usr/bin/foo mode=0755 owner=root group=bin"
for k, v in d.iteritems():
a = action.fromstr(astr.format(k))
self.assert_(action.fromstr(str(a)) == a)
self.assert_(a.hash == v)
self.assert_(k in str(a))
def test_action_tostr(self):
"""Test that actions convert to strings properly. This means
that we can feed the resulting string back into fromstr() and
get an identical action back."""
for s in self.act_strings:
self.debug(str(s))
a = action.fromstr(s)
s2 = str(a)
a2 = action.fromstr(s2)
if a.different(a2):
self.debug("a1 " + str(a))
self.debug("a2 " + str(a2))
self.assert_(not a.different(a2))
# The one place that invariant doesn't hold is when you specify
# the payload hash as the named attribute "hash", in which case
# the resulting re-stringification emits the payload hash as a
# positional attribute again ...
s = "file hash=abc123 path=usr/bin/foo mode=0755 owner=root group=bin"
self.debug(s)
a = action.fromstr(s)
s2 = str(a)
self.assert_(s2.startswith("file abc123 "))
self.assert_("hash=abc123" not in s2)
a2 = action.fromstr(s2)
self.assert_(not a.different(a2))
# ... unless of course the hash can't be represented that way.
d = {
"hash=abc=123": "abc=123",
"hash=\"one with spaces\"": "one with spaces",
"hash='one with \" character'": 'one with " character',
"hash=\"'= !@$%^\)(*\"": "'= !@$%^\)(*",
"""hash="\\"'= \\ " """:""""'= \\ """,
'\\' : '\\'
}
astr = "file %s path=usr/bin/foo mode=0755 owner=root group=bin"
for k, v in d.iteritems():
a = action.fromstr(astr % k)
self.assert_(action.fromstr(str(a)) == a)
self.assert_(a.hash == v)
self.assert_(k in str(a))
def __mangle_license(self, fmri):
repo = self.dc.get_repo()
m_path = repo.manifest(fmri)
with open(m_path, "rb") as fh:
fmri_lines = fh.readlines()
with open(m_path, "wb") as fh:
a = None
for l in fmri_lines:
if "license=copyright" in l:
continue
elif "path=etc/bronze1" in l:
a = actions.fromstr(l)
fh.write(l)
self.assert_(a)
l = """\
license %(hash)s license=foo chash=%(chash)s pkg.csize=%(csize)s \
pkg.size=%(size)s""" % {
"hash":a.hash,
"chash":a.attrs["chash"],
"csize":a.attrs["pkg.csize"],
"size":a.attrs["pkg.size"]
}
fh.write(l)
repo.rebuild()
def gen_actions_by_type(self, atype, excludes=EmptyI):
""" generate actions of the specified type;
use already in-memory stuff if already loaded,
otherwise use per-action types files"""
if self.loaded: #if already loaded, use in-memory cached version
# invoke subclass method to generate action by action
for a in Manifest.gen_actions_by_type(self, atype,
excludes):
yield a
return
mpath = self.__file_path("manifest.dircache")
if not os.path.exists(mpath):
# no cached copy :-(
if not self.loaded:
# get manifest from disk
self.__load()
# invoke subclass method to generate action by action
for a in Manifest.gen_actions_by_type(self, atype,
excludes):
yield a
else:
# we have a cached copy - use it
mpath = self.__file_path("manifest.%s" % atype)
if not os.path.exists(mpath):
return # no such action in this manifest
f = file(mpath)
for l in f:
a = actions.fromstr(l.strip())
if a.include_this(excludes):
yield a
f.close()
def test_action_tostr(self):
"""Test that actions convert to strings properly. This means
that we can feed the resulting string back into fromstr() and
get an identical action back (excluding a few cases detailed in
the test)."""
for s in self.act_strings:
self.debug(str(s))
a = action.fromstr(s)
s2 = str(a)
a2 = action.fromstr(s2)
if a.different(a2):
self.debug("a1 " + str(a))
self.debug("a2 " + str(a2))
self.assertTrue(not a.different(a2))
# The first case that invariant doesn't hold is when you specify
# the payload hash as the named attribute "hash", in which case
# the resulting re-stringification emits the payload hash as a
# positional attribute again ...
s = "file hash=abc123 path=usr/bin/foo mode=0755 owner=root group=bin"
self.debug(s)
a = action.fromstr(s)
s2 = str(a)
self.assertTrue(s2.startswith("file abc123 "))
self.assertTrue("hash=abc123" not in s2)
a2 = action.fromstr(s2)
self.assertTrue(not a.different(a2))
# ... unless of course the hash can't be represented that way.
d = {
"hash=abc=123": "abc=123",
"hash=\"one with spaces\"": "one with spaces",
"hash='one with \" character'": 'one with " character',
"hash=\"'= !@$%^\)(*\"": "'= !@$%^\)(*",
"""hash="\\"'= \\ " """:""""'= \\ """,
'\\' : '\\'
}
astr = "file {0} path=usr/bin/foo mode=0755 owner=root group=bin"
for k, v in six.iteritems(d):
a = action.fromstr(astr.format(k))
self.assertTrue(action.fromstr(str(a)) == a)
self.assertTrue(a.hash == v)
self.assertTrue(k in str(a))
# The attributes are verified separately from the stringified
# action in the tests below to ensure that the attributes were
# parsed independently and not as a single value (e.g.
# 'file path=etc/foo\nfacet.debug=true' is parsed as having a
# path attribute and a facet.debug attribute).
# The next case that invariant doesn't hold is when you have
# multiple, quoted values for a single attribute (this case
# primarily exists for use with line-continuation support
# offered by the Manifest class).
expected = 'set name=pkg.description value="foo bar baz"'
expattrs = { 'name': 'pkg.description', 'value': 'foo bar baz' }
for astr in (
"set name=pkg.description value='foo ''bar ''baz'",
"set name=pkg.description value='foo ' 'bar ' 'baz'",
'set name=pkg.description value="foo " "bar " "baz"'):
self.__assert_action_str(astr, expected, expattrs)
expected = "set name=pkg.description value='foo \"bar\" baz'"
expattrs = { 'name': 'pkg.description',
'value': 'foo "bar" baz' }
for astr in (
"set name=pkg.description value='foo \"bar\" ''baz'",
"set name=pkg.description value='foo \"bar\" '\"baz\""):
self.__assert_action_str(astr, expected, expattrs)
# The next case that invariant doesn't hold is when there are
# multiple whitespace characters between attributes or after the
# action type.
expected = 'set name=pkg.description value=foo'
expattrs = { 'name': 'pkg.description', 'value': 'foo' }
for astr in (
"set name=pkg.description value=foo",
"set name=pkg.description value=foo",
"set name=pkg.description value=foo",
"set\n name=pkg.description \nvalue=foo",
"set\t\nname=pkg.description\t\nvalue=foo"):
# To force stressing the parsing logic a bit more, we
# parse an action with a multi-value attribute that
# needs concatention each time before we parse a
# single-value attribute that needs concatenation.
#
# This simulates a refcount bug that was found during
# development and serves as an extra stress-test.
self.__assert_action_str(
'set name=multi-value value=bar value="foo ""baz"',
'set name=multi-value value=bar value="foo baz"',
{ 'name': 'multi-value',
'value': ['bar', 'foo baz'] })
self.__assert_action_str(astr, expected, expattrs)
astr = 'file path=etc/foo\nfacet.debug=true'
expected = 'file NOHASH facet.debug=true path=etc/foo'
expattrs = { 'path': 'etc/foo', 'facet.debug': 'true' }
self.__assert_action_str(astr, expected, expattrs)
def __assert_action_str(self, astr, expected, expattrs):
"""Private helper function for action stringification
testing."""
act = action.fromstr(astr)
self.assertEqualDiff(expected, str(act))
self.assertEqualDiff(expattrs, act.attrs)
def test_action_errors(self):
# Unknown action type
self.assertRaises(action.UnknownActionError, action.fromstr,
"moop bar=baz")
self.assertRaises(action.UnknownActionError, action.fromstr,
"setbar=baz quux=quark")
# Nothing but the action type or type is malformed.
self.assertMalformed("moop")
self.assertMalformed("setbar=baz")
# Bad quoting: missing close quote
self.assertMalformed("file 12345 path=/tmp/foo name=\"foo bar")
self.assertMalformed("file 12345 path=/tmp/foo name=\"foo bar\\")
# Bad quoting: quote in key
self.assertMalformed("file 12345 path=/tmp/foo \"name=foo bar")
self.assertMalformed("file 12345 path=/tmp/foo na\"me=foo bar")
self.assertMalformed("file 1234 path=/tmp/foo \"foo\"=bar")
# Missing key
self.assertMalformed("file 1234 path=/tmp/foo =\"\"")
self.assertMalformed("file path=/tmp/foo =")
self.assertMalformed("file 1234 path=/tmp/foo =")
self.assertMalformed("file 1234 path=/tmp/foo ==")
self.assertMalformed("file 1234 path=/tmp/foo ===")
# Missing value
self.assertMalformed("file 1234 path=/tmp/foo broken=")
self.assertMalformed("file 1234 path=/tmp/foo broken= ")
self.assertMalformed("file 1234 path=/tmp/foo broken")
# Whitespace in key
self.assertMalformed("file 1234 path=/tmp/foo bro ken")
# Attribute value is invalid.
self.assertInvalid("depend type=unknown [email protected]")
# Missing required attribute 'type'.
self.assertInvalid("depend [email protected]")
# Missing key attribute 'fmri'.
self.assertInvalid("depend type=require")
# Mutiple fmri values only allowed for require-any deps.
self.assertInvalid("depend type=require fmri=foo fmri=bar")
# 'path' attribute specified multiple times
self.assertInvalid("file 1234 path=foo path=foo mode=777 owner=root group=root")
self.assertInvalid("link path=foo path=foo target=link")
self.assertInvalid("dir path=foo path=foo mode=777 owner=root group=root")
# 'data' used as an attribute key
self.assertInvalid("file 1234 path=/tmp/foo data=rubbish")
# Missing required attribute 'path'.
self.assertRaises(action.InvalidActionError, action.fromstr,
"file 1234 owner=foo")
# Missing required attribute 'name'.
self.assertRaises(action.InvalidActionError, action.fromstr,
"driver alias=pci1234,56 alias=pci4567,89 class=scsi")
# Verify malformed actions > 255 characters don't cause corrupt
# exception action strings.
self.assertMalformed("""legacy arch=i386 category=GNOME2,application,JDSosol desc="XScreenSaver is two things: it is both a large collection of screen savers (distributed in the "hacks" packages) and it is also the framework for blanking and locking the screen (this package)." hotline="Please contact your local service provider" name="XScreenSaver is two things: it is both a large collection of screen savers (distributed in the "hacks" packages) and it is also the framework for blanking and locking the screen (this package)." pkg=SUNWxscreensaver vendor="XScreenSaver Community" version=5.11,REV=110.0.4.2010.07.08.22.18""")
self.assertMalformed("""legacy arch=i386 category=GNOME2,application,JDSosol desc="XScreenSaver is two things: it is both a large collection of screen savers (distributed in the "hacks" packages) and it is also the framework for blanking and locking the screen (this package)." hotline="Please contact your local service provider" name="XScreenSaver is two things: it is both a large collection of screen savers (distributed in the "hacks" packages) and it is also the framework for blanking and locking the screen (this package)." pkg=SUNWxscreensaver-l10n vendor="XScreenSaver Community" version=5.11,REV=110.0.4.2010.07.08.22.18""")
# Missing required attribute 'algorithm'.
self.assertRaises(action.InvalidActionError, action.fromstr,
"signature 12345 pkg.cert=bar")
# The payload hash can't be specified as both a named and a
# positional attribute if they're not identical.
self.assertInvalid("file xyz789 hash=abc123 path=usr/bin/foo mode=0755 owner=root group=bin")
action.fromstr("file abc123 hash=abc123 path=usr/bin/foo mode=0755 owner=root group=bin")
def test_action_parser(self):
action.fromstr("file 12345 name=foo path=/tmp/foo")
action.fromstr("file 12345 name=foo attr=bar path=/tmp/foo")
action.fromstr("file 12345 name=foo attr=bar attr=bar path=/tmp/foo")
action.fromstr("file 12345 name=foo path=/tmp/foo attr=bar")
action.fromstr("file 12345 name=foo path=/tmp/foo attr=bar ")
action.fromstr("file 12345 name=foo path=/tmp/foo attr=bar ")
action.fromstr("file 12345 name=\"foo bar\" path=\"/tmp/foo\" attr=\"bar baz\"")
action.fromstr("file 12345 name=\"foo bar\" path=\"/tmp/foo\" attr=\"bar baz\"")
action.fromstr("file 12345 name=foo value=barbaz path=/tmp/foo")
action.fromstr("file 12345 name=foo value=\"bar baz\" path=/tmp/foo")
action.fromstr("file 12345 name=\"foo bar\" value=baz path=/tmp/foo")
action.fromstr("file 12345 name=foo value=barbazquux path=/tmp/foo")
action.fromstr("file 12345 name=foo value=\"bar baz quux\" path=/tmp/foo")
action.fromstr("file 12345 name=\"foo bar baz\" value=quux path=/tmp/foo")
action.fromstr("file 12345 name=\"foo\" value=\"bar\" path=/tmp/foo")
action.fromstr("file 12345 name=foo value=\"bar\" path=/tmp/foo")
action.fromstr("file 12345 name=\"foo\" value=bar path=/tmp/foo")
action.fromstr("signature 12345 algorithm=foo")
# For convenience, we allow set actions to be expressed as
# "<name>=<value>", rather than "name=<name> value=<value>", but
# we always convert to the latter. Verify that both forms are
# parsed as expected.
a = action.fromstr("set pkg.obsolete=true")
a2 = action.fromstr("set name=pkg.obsolete value=true")
self.assertEqual(str(a), str(a2))
self.assertAttributes(a, ["name", "value"])
self.assertAttributeValue(a, "name", "pkg.obsolete")
self.assertAttributeValue(a, "value", "true")
# Single quoted value
a = action.fromstr("file 12345 name='foo' value=bar path=/tmp/foo")
self.assertAttributes(a, ["name", "value", "path"])
self.assertAttributeValue(a, "name", "foo")
self.assertAttributeValue(a, "value", "bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# Make sure that unescaped double quotes are parsed properly
# inside a single-quoted value.
a = action.fromstr("file 12345 name='f\"o\"o' value=bar path=/tmp/foo")
self.assertAttributes(a, ["name", "path", "value"])
self.assertAttributeValue(a, "name", "f\"o\"o")
self.assertAttributeValue(a, "value", "bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# Make sure that escaped single quotes are parsed properly
# inside a single-quoted value.
a = action.fromstr("file 12345 name='f\\'o\\'o' value=bar path=/tmp/foo")
self.assertAttributes(a, ["name", "path", "value"])
self.assertAttributeValue(a, "name", "f'o'o")
self.assertAttributeValue(a, "value", "bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# You should be able to separate key/value pairs with tabs as
# well as spaces.
a = action.fromstr("file 12345 name=foo\tvalue=bar path=/tmp/foo")
self.assertAttributes(a, ["name", "path", "value"])
self.assertAttributeValue(a, "name", "foo")
self.assertAttributeValue(a, "value", "bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# Unescaped, unpaired quotes are allowed in the middle of values
# without further quoting
a = action.fromstr("file 12345 name=foo\"bar path=/tmp/foo")
self.assertAttributes(a, ["name", "path"])
self.assertAttributeValue(a, "name", "foo\"bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# They can even be paired. Note this is not like shell quoting.
a = action.fromstr("file 12345 name=foo\"bar\"baz path=/tmp/foo")
self.assertAttributes(a, ["name", "path"])
self.assertAttributeValue(a, "name", "foo\"bar\"baz")
self.assertAttributeValue(a, "path", "tmp/foo")
# An unquoted value can end in an escaped backslash
a = action.fromstr("file 12345 name=foo\\ path=/tmp/foo")
self.assertAttributes(a, ["name", "path"])
self.assertAttributeValue(a, "name", "foo\\")
self.assertAttributeValue(a, "path", "tmp/foo")
# An action with multiple identical attribute names should
# result in an attribute with a list value.
a = action.fromstr("driver alias=pci1234,56 alias=pci4567,89 class=scsi name=lsimega")
self.assertAttributes(a, ["alias", "class", "name"])
self.assertAttributeValue(a, "alias", ["pci1234,56", "pci4567,89"])
# An action with an empty value.
a = action.fromstr('set name=foo value=""')
self.assertAttributes(a, ["name", "value"])
self.assertAttributeValue(a, "name", "foo")
self.assertAttributeValue(a, "value", "")
# An action with an empty value as part of a list.
a = action.fromstr('set name=foo value=ab value="" value=c')
self.assertAttributes(a, ["name", "value"])
self.assertAttributeValue(a, "name", "foo")
self.assertAttributeValue(a, "value", ["ab", "c", ""])
# An action with its key attribute and extra attributes that
# are not used by the package system.
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" custom="foo" '
'bool_val=true')
self.assertAttributes(a, ["license", "custom", "bool_val"])
self.assertAttributeValue(a, "license", 'Common Development '
'and Distribution License 1.0 (CDDL)')
self.assertAttributeValue(a, "custom", "foo")
self.assertAttributeValue(a, "bool_val", "true")
# Really long actions with lots of backslash-escaped quotes
# should work.
a = action.fromstr(r'set name=pkg.description value="Sphinx is a tool that makes it easy to create intelligent \"and beautiful documentation f\"or Python projects (or \"other documents consisting of\" multiple reStructuredText so\"urces), written by Georg Bran\"dl. It was originally created\" to translate the new Python \"documentation, but has now be\"en cleaned up in the hope tha\"t it will be useful to many o\"ther projects. Sphinx uses re\"StructuredText as its markup \"language, and many of its str\"engths come from the power an\"d straightforwardness of reSt\"ructuredText and its parsing \"and translating suite, the Do\"cutils. Although it is still \"under constant development, t\"he following features are alr\"eady present, work fine and c\"an be seen \"in action\" \"in the Python docs: * Output \"formats: HTML (including Wind\"ows HTML Help) and LaTeX, for\" printable PDF versions * Ext\"ensive cross-references: sema\"ntic markup and automatic lin\"ks for functions, classes, gl\"ossary terms and similar piec\"es of information * Hierarchi\"cal structure: easy definitio\"n of a document tree, with au\"tomatic links to siblings, pa\"rents and children * Automati\"c indices: general index as w\"ell as a module index * Code \"handling: automatic highlight\"ing using the Pygments highli\"ghter * Various extensions ar\"e available, e.g. for automat\"ic testing of snippets and in\"clusion of appropriately formatted docstrings."')
self.assertTrue(a.attrs["value"].count('"') == 45)
# Make sure that the hash member of the action object properly
# contains the value of the "hash" named attribute.
a = action.fromstr("file hash=abc123 path=usr/bin/foo mode=0755 owner=root group=bin")
self.assertTrue(a.hash == "abc123")
def assert_invalid_attrs(astr):
bad_act = action.fromstr(astr)
try:
bad_act.validate()
except Exception, e:
self.debug(str(e))
def search_dict(file_path, excludes, return_line=False, log=None):
"""Produces the search dictionary for a specific manifest.
A dictionary is constructed which maps a tuple of token,
action type, key, and the value that matched the token to
the byte offset into the manifest file. file_path is the
path to the manifest file. excludes is the variants which
should be allowed in this image. return_line is a debugging
flag which makes the function map the information to the
string of the line, rather than the byte offset to allow
easier debugging."""
if log is None:
log = lambda x: None
file_handle = file(file_path)
cur_pos = 0
line = file_handle.readline()
action_dict = {}
def __handle_list(lst, cp):
"""Translates what actions.generate_indices produces
into a dictionary mapping token, action_name, key, and
the value that should be displayed for matching that
token to byte offsets into the manifest file.
The "lst" parameter is the data to be converted.
The "cp" parameter is the byte offset into the file
for the action which produced lst."""
for action_name, subtype, tok, full_value in lst:
if action_name == "set":
if full_value is None:
full_value = tok
else:
if full_value is None:
full_value = subtype
if full_value is None:
full_value = action_name
if isinstance(tok, list):
__handle_list([(action_name, subtype, t, full_value)
for t in tok], cp)
else:
if (tok, action_name, subtype, full_value) in action_dict:
action_dict[(tok, action_name, subtype,
full_value)].append(cp)
else:
action_dict[(tok, action_name, subtype,
full_value)] = [cp]
while line:
l = line.strip()
if l and l[0] != "#":
try:
action = actions.fromstr(l)
except actions.ActionError, e:
log((_("%(fp)s:\n%(e)s") % {"fp": file_path, "e": e}))
else:
if action.include_this(excludes):
if action.attrs.has_key("path"):
np = action.attrs["path"].lstrip(os.path.sep)
action.attrs["path"] = \
np
try:
inds = action.generate_indices()
except KeyError, k:
log(
_("%(fp)s contains "
"an action which is"
" missing the "
"expected attribute"
": %(at)s.\nThe "
"action is:"
"%(act)s") % {
"fp": file_path,
"at": k.args[0],
"act": l
})
else:
arg = cur_pos
if return_line:
arg = l
__handle_list(inds, arg)
def test_validate(self):
"""Verify that action validate() works as expected; currently
only used during publication or action execution failure."""
fact = "file 12345 name=foo path=/tmp/foo mode=XXX"
dact = "dir path=/tmp mode=XXX"
def assertActionError(astr,
error=action.InvalidActionAttributesError):
bad_act = action.fromstr(astr)
try:
bad_act.validate()
except Exception as e:
self.debug(str(e))
else:
self.debug("expected failure validating: {0}".format(
astr))
self.assertRaises(error, bad_act.validate)
# Verify mode attributes for file specified more than once are
# rejected.
nact = "file path=/usr/bin/foo owner=root group=root " \
"mode=0555 mode=0555"
assertActionError(nact)
# Verify predicate and target attributes of FMRIs must be valid.
for nact in (
# FMRI value is invalid.
"depend type=require-any fmri=foo fmri=bar fmri=invalid@abc",
# Missing required attribute 'type'.
"depend [email protected]"
# type is invalid.
"depend type=unknown [email protected]",
# Multiple values never allowed for depend action 'type' attribute.
"depend type=require type=require-any fmri=foo",
# Mutiple fmri values only allowed for require-any deps.
"depend type=require fmri=foo fmri=bar",
# Predicate is missing for conditional dependency.
"depend type=conditional fmri=foo",
# Predicate value is invalid.
"depend type=conditional predicate=-invalid fmri=foo",
# Predicate isn't valid for dependency type.
"depend type=require predicate=1invalid fmri=foo",
# root-image attribute is only valid for origin dependencies.
"depend type=require fmri=foo root-image=true",
# Multiple values for predicate are not allowed.
"depend type=conditional predicate=foo predicate=bar fmri=baz",
# Multiple values for ignore-check are not allowed.
"depend type=require fmri=foo ignore-check=true ignore-check=false"):
assertActionError(nact)
# Verify multiple values for file attributes are rejected.
for attr in ("pkg.size", "pkg.csize", "chash", "preserve",
"overlay", "elfhash", "original_name", "facet.doc",
"owner", "group", "preserve-version"):
nact = "file path=/usr/bin/foo owner=root group=root " \
"mode=0555 {attr}=1 {attr}=2 {attr}=3".format(
attr=attr)
assertActionError(nact)
# Verify invalid values are not allowed for mode attribute on
# file and dir actions.
for act in (fact, dact):
for bad_mode in ("", 'mode=""', "mode=???",
"mode=44755", "mode=44", "mode=999", "mode=0898"):
nact = act.replace("mode=XXX", bad_mode)
assertActionError(nact)
# Verify multiple values aren't allowed for legacy action
# attributes.
for attr in ("category", "desc", "hotline", "name", "vendor",
"version"):
nact = "legacy pkg=SUNWcs {attr}=1 {attr}=2".format(
attr=attr)
assertActionError(nact)
# Verify multiple values aren't allowed for gid of group.
nact = "group groupname=staff gid=100 gid=101"
assertActionError(nact)
# Verify only numeric value is allowed for gid of group.
nact = "group groupname=staff gid=abc"
assertActionError(nact)
# Verify multiple values are not allowed for must-accept and
# must-display attributes of license actions.
for attr in ("must-accept", "must-display"):
nact = "license license=copyright {attr}=true " \
"{attr}=false".format(attr=attr)
assertActionError(nact)
# Ensure link and hardlink attributes are validated properly.
for aname in ("link", "hardlink"):
# Action with mediator without mediator properties is
# invalid.
nact = "{0} path=usr/bin/vi target=../sunos/bin/edit " \
"mediator=vi".format(aname)
assertActionError(nact)
# Action with multiple mediator values is invalid.
nact = "{0} path=usr/bin/vi target=../sunos/bin/edit " \
"mediator=vi mediator=vim " \
"mediator-implementatio=svr4".format(aname)
assertActionError(nact)
# Action with mediator properties without mediator
# is invalid.
props = {
"mediator-version": "1.0",
"mediator-implementation": "svr4",
"mediator-priority": "site",
}
for prop, val in six.iteritems(props):
nact = "{0} path=usr/bin/vi " \
"target=../sunos/bin/edit {1}={2}".format(aname,
prop, val)
assertActionError(nact)
# Action with multiple values for any property is
# invalid.
for prop, val in six.iteritems(props):
nact = "{0} path=usr/bin/vi " \
"target=../sunos/bin/edit mediator=vi " \
"{1}={2} {3}={4} ".format(aname, prop, val, prop,
val)
if prop == "mediator-priority":
# mediator-priority alone isn't
# valid, so test multiple value
# invalid, add something.
nact += " mediator-version=1.0"
assertActionError(nact)
# Verify invalid mediator names are rejected.
for value in ("not/valid", "not valid", "not.valid"):
nact = "{0} path=usr/bin/vi target=vim " \
"mediator=\"{1}\" mediator-implementation=vim" \
.format(aname, value)
assertActionError(nact)
# Verify invalid mediator-versions are rejected.
for value in ("1.a", "abc", ".1"):
nact = "{0} path=usr/bin/vi target=vim " \
"mediator=vim mediator-version={1}" \
.format(aname, value)
assertActionError(nact)
# Verify invalid mediator-implementations are rejected.
for value in ("1.a", "@", "@1", "[email protected]",
"vim@abc"):
nact = "{0} path=usr/bin/vi target=vim " \
"mediator=vim mediator-implementation={1}" \
.format(aname, value)
assertActionError(nact)
# Verify multiple targets are not allowed.
nact = "{0} path=/usr/bin/foo target=bar target=baz".format(
aname)
assertActionError(nact)
# Verify multiple values are not allowed for set actions such as
# pkg.description, pkg.obsolete, pkg.renamed, and pkg.summary.
for attr in ("pkg.description", "pkg.obsolete", "pkg.renamed",
"pkg.summary", "pkg.depend.explicit-install"):
nact = "set name={0} value=true value=false".format(attr)
assertActionError(nact)
# Verify signature action attribute 'value' is required during
# publication.
nact = "signature 12345 algorithm=foo"
assertActionError(nact)
# Verify multiple values aren't allowed for user attributes.
for attr in ("password", "group", "gcos-field", "home-dir",
"login-shell", "ftpuser"):
nact = "user username=user {attr}=ab {attr}=cd ".format(
attr=attr)
assertActionError(nact)
for attr in ("uid", "lastchg", "min","max", "warn", "inactive",
"expire", "flag"):
nact = "user username=user {attr}=1 {attr}=2".format(
attr=attr)
assertActionError(nact)
# Verify only numeric values are allowed for user attributes
# expecting a number.
for attr in ("uid", "lastchg", "min","max", "warn", "inactive",
"expire", "flag"):
nact = "user username=user {0}=abc".format(attr)
assertActionError(nact)
# Malformed pkg actuators
assertActionError(
"set name=pkg.additional-update-on-uninstall "
"value=&@M")
assertActionError(
"set name=pkg.additional-update-on-uninstall "
"value=A@1 value=&@M")
# Unknown actuator (should pass)
act = action.fromstr(
"set name=pkg.additional-update-on-update value=A@1")
act.validate()
os.makedirs(self.dir)
except EnvironmentError, e:
if e.errno == errno.EEXIST:
raise TransactionAlreadyOpenError(trans_basename)
raise TransactionOperationError(e)
#
# always create a minimal manifest
#
tfile = file("%s/manifest" % self.dir, "ab+")
# Build a set action containing the fully qualified FMRI and add
# it to the manifest. While it may seem inefficient to create
# an action string, convert it to an action, and then back, it
# does ensure that the server is adding a valid action.
fact = actions.fromstr("set name=pkg.fmri value=%s" % self.fmri)
print >> tfile, str(fact)
tfile.close()
# XXX:
# validate that this version can be opened
# if we specified no release, fail
# if we specified a release without branch, open next branch
# if we specified a release with branch major, open same
# branch minor
# if we specified a release with branch major and minor, use
# as specified
# we should disallow new package creation, if so flagged
# if not found, create package
# set package state to TRANSACTING
# Like the unix utility comm, except that this function
# takes an arbitrary number of manifests and compares them,
# returning a tuple consisting of each manifest's actions
# that are not the same for all manifests, followed by a
# list of actions that are the same in each manifest.
try:
action_lists = list(manifest.Manifest.comm(manifest_list))
except manifest.ManifestError, e:
raise PkgmergeException(
"Duplicate action(s) in package \"%s\": \n%s" %
(new_fmri.pkg_name, e))
# Declare new package FMRI.
action_lists[-1].insert(0,
actions.fromstr("set name=pkg.fmri value=%s" % new_fmri))
for a_list, v in zip(action_lists[:-1], variant_list):
for a in a_list:
a.attrs[variant] = v
# discard any blend tags for this variant from common list
for a in action_lists[-1]:
blend_attrs = set(a.attrlist("pkg.merge.blend"))
match = blend_names & blend_attrs
for m in list(match):
if len(blend_attrs) == 1:
del a.attrs["pkg.merge.blend"]
else:
a.attrlist("pkg.merge.blend").remove(m)
# combine actions into single list
allactions = reduce(lambda a, b: a + b, action_lists)
if e.errno == errno.EEXIST:
raise TransactionAlreadyOpenError(
trans_basename)
raise TransactionOperationError(e)
#
# always create a minimal manifest
#
tfpath = os.path.join(self.dir, "manifest")
tfile = file(tfpath, "ab+")
# Build a set action containing the fully qualified FMRI and add
# it to the manifest. While it may seem inefficient to create
# an action string, convert it to an action, and then back, it
# does ensure that the server is adding a valid action.
fact = actions.fromstr("set name=pkg.fmri value=%s" % self.fmri)
print >> tfile, str(fact)
tfile.close()
# XXX:
# validate that this version can be opened
# if we specified no release, fail
# if we specified a release without branch, open next branch
# if we specified a release with branch major, open same
# branch minor
# if we specified a release with branch major and minor, use
# as specified
# we should disallow new package creation, if so flagged
# if not found, create package
# set package state to TRANSACTING
def set_content(self, content, excludes=EmptyI):
"""content is the text representation of the manifest"""
self.actions = []
self.actions_bytype = {}
self.variants = {}
self.facets = {}
self.attributes = {}
# So we could build up here the type/key_attr dictionaries like
# sdict and odict in difference() above, and have that be our
# main datastore, rather than the simple list we have now. If
# we do that here, we can even assert that the "same" action
# can't be in a manifest twice. (The problem of having the same
# action more than once in packages that can be installed
# together has to be solved somewhere else, though.)
for l in content.splitlines():
l = l.lstrip()
if not l or l[0] == "#":
continue
try:
action = actions.fromstr(l)
except actions.ActionError, e:
# Add the FMRI to the exception and re-raise
e.fmri = self.fmri
raise
# XXX handle legacy transition issues; not needed after
# 2009.06 release & republication are complete.
if "opensolaris.zone" in action.attrs and \
"variant.opensolaris.zone" not in action.attrs:
action.attrs["variant.opensolaris.zone"] = \
action.attrs["opensolaris.zone"]
if action.name == "set" and \
action.attrs["name"] == "authority":
# Translate old action to new.
action.attrs["name"] = "publisher"
if action.attrs.has_key("path"):
np = action.attrs["path"].lstrip(os.path.sep)
action.attrs["path"] = np
if not action.include_this(excludes):
continue
self.actions.append(action)
if action.name not in self.actions_bytype:
self.actions_bytype[action.name] = [action]
else:
self.actions_bytype[action.name].append(action)
# add any set actions to attributes
if action.name == "set":
self.fill_attributes(action)
# append any variants and facets to manifest dict
v_list, f_list = action.get_varcet_keys()
if v_list or f_list:
for v, d in zip(v_list, repeat(self.variants)) \
+ zip(f_list, repeat(self.facets)):
if v not in d:
d[v] = set([action.attrs[v]])
else:
d[v].add(action.attrs[v])
def set_content(self, content, excludes=EmptyI):
"""content is the text representation of the manifest"""
self.actions = []
self.actions_bytype = {}
self.variants = {}
self.facets = {}
self.attributes = {}
# So we could build up here the type/key_attr dictionaries like
# sdict and odict in difference() above, and have that be our
# main datastore, rather than the simple list we have now. If
# we do that here, we can even assert that the "same" action
# can't be in a manifest twice. (The problem of having the same
# action more than once in packages that can be installed
# together has to be solved somewhere else, though.)
for l in content.splitlines():
l = l.lstrip()
if not l or l[0] == "#":
continue
try:
action = actions.fromstr(l)
except actions.ActionError, e:
# Add the FMRI to the exception and re-raise
e.fmri = self.fmri
raise
# XXX handle legacy transition issues; not needed after
# 2009.06 release & republication are complete.
if "opensolaris.zone" in action.attrs and \
"variant.opensolaris.zone" not in action.attrs:
action.attrs["variant.opensolaris.zone"] = \
action.attrs["opensolaris.zone"]
if action.name == "set" and \
action.attrs["name"] == "authority":
# Translate old action to new.
action.attrs["name"] = "publisher"
if action.attrs.has_key("path"):
np = action.attrs["path"].lstrip(os.path.sep)
action.attrs["path"] = np
if not action.include_this(excludes):
continue
self.actions.append(action)
if action.name not in self.actions_bytype:
self.actions_bytype[action.name] = [ action ]
else:
self.actions_bytype[action.name].append(action)
# add any set actions to attributes
if action.name == "set":
self.fill_attributes(action)
# append any variants and facets to manifest dict
v_list, f_list = action.get_varcet_keys()
if v_list or f_list:
for v, d in zip(v_list, repeat(self.variants)) \
+ zip(f_list, repeat(self.facets)):
if v not in d:
d[v] = set([action.attrs[v]])
else:
d[v].add(action.attrs[v])
def test_action_errors(self):
# Unknown action type
self.assertRaises(action.UnknownActionError, action.fromstr,
"moop bar=baz")
self.assertRaises(action.UnknownActionError, action.fromstr,
"setbar=baz quux=quark")
# Nothing but the action type or type is malformed.
self.assertMalformed("moop")
self.assertMalformed("setbar=baz")
# Bad quoting: missing close quote
self.assertMalformed("file 12345 path=/tmp/foo name=\"foo bar")
self.assertMalformed("file 12345 path=/tmp/foo name=\"foo bar\\")
# Bad quoting: quote in key
self.assertMalformed("file 12345 path=/tmp/foo \"name=foo bar")
self.assertMalformed("file 12345 path=/tmp/foo na\"me=foo bar")
self.assertMalformed("file 1234 path=/tmp/foo \"foo\"=bar")
# Missing key
self.assertMalformed("file 1234 path=/tmp/foo =\"\"")
self.assertMalformed("file path=/tmp/foo =")
self.assertMalformed("file 1234 path=/tmp/foo =")
self.assertMalformed("file 1234 path=/tmp/foo ==")
self.assertMalformed("file 1234 path=/tmp/foo ===")
# Missing value
self.assertMalformed("file 1234 path=/tmp/foo broken=")
self.assertMalformed("file 1234 path=/tmp/foo broken= ")
self.assertMalformed("file 1234 path=/tmp/foo broken=\t")
self.assertMalformed("file 1234 path=/tmp/foo broken=\n")
self.assertMalformed("file 1234 path=/tmp/foo broken")
# Whitespace in key
self.assertMalformed("file 1234 path=/tmp/foo bro ken")
self.assertMalformed("file 1234 path=/tmp/foo\tbro\tken")
self.assertMalformed("file 1234 path=/tmp/foo\nbro\nken")
self.assertMalformed("file 1234 path ='/tmp/foo")
self.assertMalformed("file 1234 path\t=/tmp/foo")
self.assertMalformed("file 1234 path\n=/tmp/foo")
# Missing key attribute 'fmri'.
self.assertInvalid("depend type=require")
# XXX Fails in Python 3.4 due to module import issue; see
# set_invalid_action_error in actions/_common.c.
if six.PY2:
# Multiple values not allowed for 'fmri' if 'type' is
# multi-valued.
self.assertInvalid("depend type=require "
"type=require-any fmri=foo fmri=bar")
# 'path' attribute specified multiple times
self.assertInvalid("file 1234 path=foo path=foo mode=777 owner=root group=root")
self.assertInvalid("link path=foo path=foo target=link")
self.assertInvalid("dir path=foo path=foo mode=777 owner=root group=root")
# 'data' used as an attribute key
self.assertInvalid("file 1234 path=/tmp/foo data=rubbish")
# Missing required attribute 'path'.
self.assertRaises(action.InvalidActionError, action.fromstr,
"file 1234 owner=foo")
# Missing required attribute 'name'.
self.assertRaises(action.InvalidActionError, action.fromstr,
"driver alias=pci1234,56 alias=pci4567,89 class=scsi")
# Verify malformed actions > 255 characters don't cause corrupt
# exception action strings.
self.assertMalformed("""legacy arch=i386 category=GNOME2,application,JDSosol desc="XScreenSaver is two things: it is both a large collection of screen savers (distributed in the "hacks" packages) and it is also the framework for blanking and locking the screen (this package)." hotline="Please contact your local service provider" name="XScreenSaver is two things: it is both a large collection of screen savers (distributed in the "hacks" packages) and it is also the framework for blanking and locking the screen (this package)." pkg=SUNWxscreensaver vendor="XScreenSaver Community" version=5.11,REV=110.0.4.2010.07.08.22.18""")
self.assertMalformed("""legacy arch=i386 category=GNOME2,application,JDSosol desc="XScreenSaver is two things: it is both a large collection of screen savers (distributed in the "hacks" packages) and it is also the framework for blanking and locking the screen (this package)." hotline="Please contact your local service provider" name="XScreenSaver is two things: it is both a large collection of screen savers (distributed in the "hacks" packages) and it is also the framework for blanking and locking the screen (this package)." pkg=SUNWxscreensaver-l10n vendor="XScreenSaver Community" version=5.11,REV=110.0.4.2010.07.08.22.18""")
# Missing required attribute 'algorithm'.
self.assertRaises(action.InvalidActionError, action.fromstr,
"signature 12345 pkg.cert=bar")
# The payload hash can't be specified as both a named and a
# positional attribute if they're not identical.
self.assertInvalid("file xyz789 hash=abc123 path=usr/bin/foo mode=0755 owner=root group=bin")
action.fromstr("file abc123 hash=abc123 path=usr/bin/foo mode=0755 owner=root group=bin")
def open(self, rstore, client_release, pfmri):
# Store a reference to the repository storage object.
self.rstore = rstore
if client_release is None:
raise TransactionOperationError(client_release=None,
pfmri=pfmri)
if pfmri is None:
raise TransactionOperationError(pfmri=None)
if not isinstance(pfmri, six.string_types):
pfmri = str(pfmri)
self.client_release = client_release
self.pkg_name = pfmri
self.esc_pkg_name = quote(pfmri, "")
# attempt to construct an FMRI object
try:
self.fmri = fmri.PkgFmri(self.pkg_name,
self.client_release)
except fmri.FmriError as e:
raise TransactionOperationError(e)
# Version is required for publication.
if self.fmri.version is None:
raise TransactionOperationError(fmri_version=None,
pfmri=pfmri)
# Ensure that the FMRI has been fully qualified with publisher
# information or apply the default if appropriate.
if not self.fmri.publisher:
default_pub = rstore.publisher
if not default_pub:
# A publisher is required.
raise TransactionOperationError(
publisher_required=True, pfmri=pfmri)
self.fmri.publisher = default_pub
pkg_name = self.pkg_name
pub_string = "pkg://{0}/".format(default_pub)
if not pkg_name.startswith("pkg:/"):
pkg_name = pub_string + pkg_name
else:
pkg_name = pkg_name.replace("pkg:/", pub_string)
self.pkg_name = pkg_name
self.esc_pkg_name = quote(pkg_name, "")
# record transaction metadata: opening_time, package, user
# XXX publishing with a custom timestamp may require
# authorization above the basic "can open transactions".
self.open_time = self.fmri.get_timestamp()
if self.open_time:
# Strip the timestamp information for consistency with
# the case where it was not specified.
self.pkg_name = ":".join(pfmri.split(":")[:-1])
self.esc_pkg_name = quote(self.pkg_name, "")
else:
# A timestamp was not provided; try to generate a
# unique one.
while 1:
self.open_time = datetime.datetime.utcnow()
self.fmri.set_timestamp(self.open_time)
cat = rstore.catalog
if not cat.get_entry(self.fmri):
break
time.sleep(.25)
# Check that the new FMRI's version is valid. In other words,
# the package has not been renamed or frozen for the new
# version.
if not self.rstore.valid_new_fmri(self.fmri):
raise TransactionOperationError(valid_new_fmri=False,
pfmri=pfmri)
trans_basename = self.get_basename()
self.dir = os.path.join(self.rstore.trans_root, trans_basename)
try:
os.makedirs(self.dir, misc.PKG_DIR_MODE)
except EnvironmentError as e:
if e.errno == errno.EEXIST:
raise TransactionAlreadyOpenError(
trans_basename)
raise TransactionOperationError(e)
#
# always create a minimal manifest
#
tfpath = os.path.join(self.dir, "manifest")
tfile = open(tfpath, "a+")
# Build a set action containing the fully qualified FMRI and add
# it to the manifest. While it may seem inefficient to create
# an action string, convert it to an action, and then back, it
# does ensure that the server is adding a valid action.
fact = actions.fromstr("set name=pkg.fmri value={0}".format(
self.fmri))
print(str(fact), file=tfile)
tfile.close()
def test_action_parser(self):
action.fromstr("file 12345 name=foo path=/tmp/foo")
action.fromstr("file 12345 name=foo attr=bar path=/tmp/foo")
action.fromstr("file 12345 name=foo attr=bar attr=bar path=/tmp/foo")
action.fromstr("file 12345 name=foo path=/tmp/foo attr=bar")
action.fromstr("file 12345 name=foo path=/tmp/foo attr=bar ")
action.fromstr("file 12345 name=foo path=/tmp/foo attr=bar ")
action.fromstr("file 12345 name=\"foo bar\" path=\"/tmp/foo\" attr=\"bar baz\"")
action.fromstr("file 12345 name=\"foo bar\" path=\"/tmp/foo\" attr=\"bar baz\"")
action.fromstr("file 12345 name=foo value=barbaz path=/tmp/foo")
action.fromstr("file 12345 name=foo value=\"bar baz\" path=/tmp/foo")
action.fromstr("file 12345 name=\"foo bar\" value=baz path=/tmp/foo")
action.fromstr("file 12345 name=foo value=barbazquux path=/tmp/foo")
action.fromstr("file 12345 name=foo value=\"bar baz quux\" path=/tmp/foo")
action.fromstr("file 12345 name=\"foo bar baz\" value=quux path=/tmp/foo")
action.fromstr("file 12345 name=\"foo\" value=\"bar\" path=/tmp/foo")
action.fromstr("file 12345 name=foo value=\"bar\" path=/tmp/foo")
action.fromstr("file 12345 name=\"foo\" value=bar path=/tmp/foo")
action.fromstr("signature 12345 algorithm=foo")
# For convenience, we allow set actions to be expressed as
# "<name>=<value>", rather than "name=<name> value=<value>", but
# we always convert to the latter. Verify that both forms are
# parsed as expected.
a = action.fromstr("set pkg.obsolete=true")
a2 = action.fromstr("set name=pkg.obsolete value=true")
self.assertEqual(str(a), str(a2))
self.assertAttributes(a, ["name", "value"])
self.assertAttributeValue(a, "name", "pkg.obsolete")
self.assertAttributeValue(a, "value", "true")
# Single quoted value
a = action.fromstr("file 12345 name='foo' value=bar path=/tmp/foo")
self.assertAttributes(a, ["name", "value", "path"])
self.assertAttributeValue(a, "name", "foo")
self.assertAttributeValue(a, "value", "bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# Make sure that unescaped double quotes are parsed properly
# inside a single-quoted value.
a = action.fromstr("file 12345 name='f\"o\"o' value=bar path=/tmp/foo")
self.assertAttributes(a, ["name", "path", "value"])
self.assertAttributeValue(a, "name", "f\"o\"o")
self.assertAttributeValue(a, "value", "bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# Make sure that escaped single quotes are parsed properly
# inside a single-quoted value.
a = action.fromstr("file 12345 name='f\\'o\\'o' value=bar path=/tmp/foo")
self.assertAttributes(a, ["name", "path", "value"])
self.assertAttributeValue(a, "name", "f'o'o")
self.assertAttributeValue(a, "value", "bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# You should be able to separate key/value pairs with tabs as
# well as spaces.
a = action.fromstr("file 12345 name=foo\tvalue=bar path=/tmp/foo")
self.assertAttributes(a, ["name", "path", "value"])
self.assertAttributeValue(a, "name", "foo")
self.assertAttributeValue(a, "value", "bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# Unescaped, unpaired quotes are allowed in the middle of values
# without further quoting
a = action.fromstr("file 12345 name=foo\"bar path=/tmp/foo")
self.assertAttributes(a, ["name", "path"])
self.assertAttributeValue(a, "name", "foo\"bar")
self.assertAttributeValue(a, "path", "tmp/foo")
# They can even be paired. Note this is not like shell quoting.
a = action.fromstr("file 12345 name=foo\"bar\"baz path=/tmp/foo")
self.assertAttributes(a, ["name", "path"])
self.assertAttributeValue(a, "name", "foo\"bar\"baz")
self.assertAttributeValue(a, "path", "tmp/foo")
# An unquoted value can end in an escaped backslash
a = action.fromstr("file 12345 name=foo\\ path=/tmp/foo")
self.assertAttributes(a, ["name", "path"])
self.assertAttributeValue(a, "name", "foo\\")
self.assertAttributeValue(a, "path", "tmp/foo")
# An action with multiple identical attribute names should
# result in an attribute with a list value.
a = action.fromstr("driver alias=pci1234,56 alias=pci4567,89 class=scsi name=lsimega")
self.assertAttributes(a, ["alias", "class", "name"])
self.assertAttributeValue(a, "alias", ["pci1234,56", "pci4567,89"])
# An action with an empty value.
a = action.fromstr('set name=foo value=""')
self.assertAttributes(a, ["name", "value"])
self.assertAttributeValue(a, "name", "foo")
self.assertAttributeValue(a, "value", "")
# An action with an empty value as part of a list.
a = action.fromstr('set name=foo value=ab value="" value=c')
self.assertAttributes(a, ["name", "value"])
self.assertAttributeValue(a, "name", "foo")
self.assertAttributeValue(a, "value", ["ab", "c", ""])
# An action with its key attribute and extra attributes that
# are not used by the package system.
a = action.fromstr('license license="Common Development and '
'Distribution License 1.0 (CDDL)" custom="foo" '
'bool_val=true')
self.assertAttributes(a, ["license", "custom", "bool_val"])
self.assertAttributeValue(a, "license", 'Common Development '
'and Distribution License 1.0 (CDDL)')
self.assertAttributeValue(a, "custom", "foo")
self.assertAttributeValue(a, "bool_val", "true")
# Really long actions with lots of backslash-escaped quotes
# should work.
a = action.fromstr(r'set name=pkg.description value="Sphinx is a tool that makes it easy to create intelligent \"and beautiful documentation f\"or Python projects (or \"other documents consisting of\" multiple reStructuredText so\"urces), written by Georg Bran\"dl. It was originally created\" to translate the new Python \"documentation, but has now be\"en cleaned up in the hope tha\"t it will be useful to many o\"ther projects. Sphinx uses re\"StructuredText as its markup \"language, and many of its str\"engths come from the power an\"d straightforwardness of reSt\"ructuredText and its parsing \"and translating suite, the Do\"cutils. Although it is still \"under constant development, t\"he following features are alr\"eady present, work fine and c\"an be seen \"in action\" \"in the Python docs: * Output \"formats: HTML (including Wind\"ows HTML Help) and LaTeX, for\" printable PDF versions * Ext\"ensive cross-references: sema\"ntic markup and automatic lin\"ks for functions, classes, gl\"ossary terms and similar piec\"es of information * Hierarchi\"cal structure: easy definitio\"n of a document tree, with au\"tomatic links to siblings, pa\"rents and children * Automati\"c indices: general index as w\"ell as a module index * Code \"handling: automatic highlight\"ing using the Pygments highli\"ghter * Various extensions ar\"e available, e.g. for automat\"ic testing of snippets and in\"clusion of appropriately formatted docstrings."')
self.assert_(a.attrs["value"].count('"') == 45)
# Make sure that the hash member of the action object properly
# contains the value of the "hash" named attribute.
a = action.fromstr("file hash=abc123 path=usr/bin/foo mode=0755 owner=root group=bin")
self.assert_(a.hash == "abc123")
def search_dict(file_path, excludes, return_line=False,
log=None):
"""Produces the search dictionary for a specific manifest.
A dictionary is constructed which maps a tuple of token,
action type, key, and the value that matched the token to
the byte offset into the manifest file. file_path is the
path to the manifest file. excludes is the variants which
should be allowed in this image. return_line is a debugging
flag which makes the function map the information to the
string of the line, rather than the byte offset to allow
easier debugging."""
if log is None:
log = lambda x: None
file_handle = file(file_path)
cur_pos = 0
line = file_handle.readline()
action_dict = {}
def __handle_list(lst, cp):
"""Translates what actions.generate_indices produces
into a dictionary mapping token, action_name, key, and
the value that should be displayed for matching that
token to byte offsets into the manifest file.
The "lst" parameter is the data to be converted.
The "cp" parameter is the byte offset into the file
for the action which produced lst."""
for action_name, subtype, tok, full_value in lst:
if action_name == "set":
if full_value is None:
full_value = tok
else:
if full_value is None:
full_value = subtype
if full_value is None:
full_value = action_name
if isinstance(tok, list):
__handle_list([
(action_name, subtype, t,
full_value)
for t in tok
], cp)
else:
if (tok, action_name, subtype,
full_value) in action_dict:
action_dict[(tok, action_name,
subtype, full_value)
].append(cp)
else:
action_dict[(tok, action_name,
subtype, full_value)] = [cp]
while line:
l = line.strip()
if l and l[0] != "#":
try:
action = actions.fromstr(l)
except actions.ActionError, e:
log((_("%(fp)s:\n%(e)s") %
{ "fp": file_path, "e": e }))
else:
if action.include_this(excludes):
if action.attrs.has_key("path"):
np = action.attrs["path"].lstrip(os.path.sep)
action.attrs["path"] = \
np
try:
inds = action.generate_indices()
except KeyError, k:
log(_("%(fp)s contains "
"an action which is"
" missing the "
"expected attribute"
": %(at)s.\nThe "
"action is:"
"%(act)s") %
{
"fp": file_path,
"at": k.args[0],
"act":l
})
else:
arg = cur_pos
if return_line:
arg = l
__handle_list(inds, arg)
def __merge_fmris(new_fmri, manifest_list, fmri_list, variant_list, variant):
"""Private merge implementation."""
# Remove variant tags, package variant metadata, and signatures
# from manifests since we're reassigning. This allows merging
# pre-tagged, already merged pkgs, or signed packages.
blended_actions = []
blend_names = set([variant, variant[8:]])
for j, m in enumerate(manifest_list):
deleted_count = 0
vval = variant_list[j]
for i, a in enumerate(m.actions[:]):
if a.name == "signature" or \
(a.name == "set" and a.attrs["name"] == "pkg.fmri"):
# signatures and pkg.fmri actions are no longer
# valid after merging
del m.actions[i - deleted_count]
deleted_count += 1
continue
if variant in a.attrs:
if a.attrs[variant] != vval:
# we have an already merged
# manifest; filter out actions
# for other variants
del m.actions[i - deleted_count]
deleted_count += 1
continue
else:
del a.attrs[variant]
if a.name == "set" and a.attrs["name"] == variant:
if vval not in a.attrlist("value"):
raise PkgmergeException(
_("package {pkg} is tagged as "
"not supporting {var_name} "
"{var_value}").format(
pkg=fmri_list[j],
var_name=variant,
var_value=vval))
del m.actions[i - deleted_count]
deleted_count += 1
# checking if we're supposed to blend this action
# for this variant. Handle prepended "variant.".
if blend_names & set(a.attrlist("pkg.merge.blend")):
blended_actions.append((j, a))
# add blended actions to other manifests
for j, m in enumerate(manifest_list):
for k, a in blended_actions:
if k != j:
m.actions.append(a)
# Like the unix utility comm, except that this function
# takes an arbitrary number of manifests and compares them,
# returning a tuple consisting of each manifest's actions
# that are not the same for all manifests, followed by a
# list of actions that are the same in each manifest.
try:
action_lists = list(manifest.Manifest.comm(manifest_list))
except manifest.ManifestError as e:
raise PkgmergeException(
"Duplicate action(s) in package \"{0}\": \n{1}".format(
new_fmri.pkg_name, e))
# Declare new package FMRI.
action_lists[-1].insert(0,
actions.fromstr("set name=pkg.fmri value={0}".format(new_fmri)))
for a_list, v in zip(action_lists[:-1], variant_list):
for a in a_list:
a.attrs[variant] = v
# discard any blend tags for this variant from common list
for a in action_lists[-1]:
blend_attrs = set(a.attrlist("pkg.merge.blend"))
match = blend_names & blend_attrs
for m in list(match):
if len(blend_attrs) == 1:
del a.attrs["pkg.merge.blend"]
else:
a.attrlist("pkg.merge.blend").remove(m)
# combine actions into single list
allactions = reduce(lambda a, b: a + b, action_lists)
# figure out which variants are actually there for this pkg
actual_variant_list = [
v
for m, v in zip(manifest_list, variant_list)
]
# add set action to document which variants are supported
allactions.append(actions.fromstr("set name={0} {1}".format(variant,
" ".join([
"value={0}".format(a)
for a in actual_variant_list
])
)))
allactions.sort()
m = manifest.Manifest(pfmri=new_fmri)
m.set_content(content=allactions)
return m
def merge_fmris(server_list, fmri_list, variant_list, variant, basedir,
basename, get_files):
manifest_list = [
get_manifest(s, f) for s, f in zip(server_list, fmri_list)
]
# remove variant tags and package variant metadata
# from manifests since we're reassigning...
# this allows merging pre-tagged packages
for m in manifest_list:
for i, a in enumerate(m.actions[:]):
if variant in a.attrs:
del a.attrs[variant]
if a.name == "set" and a.attrs["name"] == variant:
del m.actions[i]
action_lists = manifest.Manifest.comm(*tuple(manifest_list))
# set fmri actions require special merge logic.
set_fmris = []
for l in action_lists:
for i, a in enumerate(l):
if not (a.name == "set" and a.attrs["name"] == "pkg.fmri"):
continue
set_fmris.append(a)
del l[i]
# If set fmris are present, then only the most recent one
# and add it back to the last action list.
if set_fmris:
def order(a, b):
f1 = pkg.fmri.PkgFmri(a.attrs["value"], "5.11")
f2 = pkg.fmri.PkgFmri(b.attrs["value"], "5.11")
return cmp(f1, f2)
set_fmris.sort(cmp=order)
action_lists[-1].insert(0, set_fmris[-1])
for a_list, v in zip(action_lists[0:-1], variant_list):
for a in a_list:
a.attrs[variant] = v
# combine actions into single list
allactions = reduce(lambda a, b: a + b, action_lists)
# figure out which variants are actually there for this pkg
actual_variant_list = [
v for m, v in zip(manifest_list, variant_list) if m != null_manifest
]
print "Merging %s for %s" % (basename, actual_variant_list)
# add set action to document which variants are supported
allactions.append(
actions.fromstr(
"set name=%s %s" %
(variant, " ".join(["value=%s" % a
for a in actual_variant_list]))))
allactions.sort()
m = manifest.Manifest()
m.set_content(content=allactions)
# urlquote to avoid problems w/ fmris w/ '/' character in name
basedir = os.path.join(basedir, urllib.quote(basename, ""))
if not os.path.exists(basedir):
os.makedirs(basedir)
m_path = os.path.join(basedir, "manifest")
m.store(m_path)
for f in fmri_list:
if f:
fmri = str(f).rsplit(":", 1)[0]
break
f_file = file(os.path.join(basedir, "fmri"), "w")
f_file.write(fmri)
f_file.close()
if get_files:
# generate list of hashes for each server; last is commom
already_seen = {}
def repeated(a, d):
if a in d:
return True
d[a] = 1
return False
action_sets = [
set(
[
a
for a in action_list
if hasattr(a, "hash") and not \
repeated(a.hash, already_seen)
]
)
for action_list in action_lists
]
# remove duplicate files (save time)
for server, action_set in zip(server_list + [server_list[0]],
action_sets):
if len(action_set) > 0:
fetch_files_byaction(server, action_set, basedir)
return 0
