def fetch_files_byhash(src_uri, cshashes, destdir, keep_compressed, tracker):
"""Given a list of tuples containing content hash, and size and download
the content from src_uri into destdir."""
def valid_file(h):
# XXX this should check data digest
fname = os.path.join(destdir, h)
if os.path.exists(fname):
if keep_compressed:
sz = cshashes[h][1]
else:
sz = cshashes[h][0]
if sz == 0:
return True
try:
fs = os.stat(fname)
except:
pass
else:
if fs.st_size == sz:
return True
return False
if src_uri.startswith("file://"):
try:
r = get_repo(src_uri)
except repo.RepositoryError, e:
abort(err=e)
for h in cshashes.keys():
dest = os.path.join(destdir, h)
# Check to see if the file already exists first, so the
# user can continue interrupted pkgrecv operations.
retrieve = not valid_file(h)
try:
if retrieve and keep_compressed:
src = r.file(h)
shutil.copy(src, dest)
elif retrieve:
src = file(r.file(h), "rb")
outfile = open(dest, "wb")
gunzip_from_stream(src, outfile)
outfile.close()
except (EnvironmentError,
repo.RepositoryError), e:
try:
portable.remove(dest)
except:
pass
abort(err=e)
tracker.download_add_progress(1, cshashes[h][2])
def fetch_files_byhash(src_uri, cshashes, destdir, keep_compressed, tracker):
"""Given a list of tuples containing content hash, and size and download
the content from src_uri into destdir."""
def valid_file(h):
# XXX this should check data digest
fname = os.path.join(destdir, h)
if os.path.exists(fname):
if keep_compressed:
sz = cshashes[h][1]
else:
sz = cshashes[h][0]
if sz == 0:
return True
try:
fs = os.stat(fname)
except:
pass
else:
if fs.st_size == sz:
return True
return False
if src_uri.startswith("file://"):
try:
r = get_repo(src_uri)
except repo.RepositoryError, e:
abort(err=e)
for h in cshashes.keys():
dest = os.path.join(destdir, h)
# Check to see if the file already exists first, so the
# user can continue interrupted pkgrecv operations.
retrieve = not valid_file(h)
try:
if retrieve and keep_compressed:
src = r.file(h)
shutil.copy(src, dest)
elif retrieve:
src = file(r.file(h), "rb")
outfile = open(dest, "wb")
gunzip_from_stream(src, outfile)
outfile.close()
except (EnvironmentError, repo.RepositoryError), e:
try:
portable.remove(dest)
except:
pass
abort(err=e)
tracker.download_add_progress(1, cshashes[h][2])
def __licenses(self, mfst):
"""Private function. Returns the license info from the
manifest mfst."""
license_lst = []
for lic in mfst.gen_actions_by_type("license"):
s = StringIO.StringIO()
lpath = self._depot.repo.file(lic.hash, pub=self._pub)
lfile = file(lpath, "rb")
misc.gunzip_from_stream(lfile, s, ignore_hash=True)
text = s.getvalue()
s.close()
license_lst.append(LicenseInfo(mfst.fmri, lic, text=text))
return license_lst
def __licenses(self, mfst):
"""Private function. Returns the license info from the
manifest mfst."""
license_lst = []
for lic in mfst.gen_actions_by_type("license"):
s = StringIO.StringIO()
lpath = self._depot.repo.cache_store.lookup(lic.hash)
lfile = file(lpath, "rb")
misc.gunzip_from_stream(lfile, s)
text = s.getvalue()
s.close()
license_lst.append(LicenseInfo(mfst.fmri, lic,
text=text))
return license_lst
def _verify_content(action, filepath):
"""If action contains an attribute that has the compressed
hash, read the file specified in filepath and verify
that the hash values match. If the values do not match,
remove the file and raise an InvalidContentException."""
chash = action.attrs.get("chash", None)
path = action.attrs.get("path", None)
if not chash:
# Compressed hash doesn't exist. Decompress and
# generate hash of uncompressed content.
ifile = open(filepath, "rb")
ofile = open(os.devnull, "wb")
try:
fhash = misc.gunzip_from_stream(ifile, ofile)
except zlib.error, e:
s = os.stat(filepath)
os.remove(filepath)
raise tx.InvalidContentException(
path, "zlib.error:%s" % (" ".join([str(a) for a in e.args])), size=s.st_size
)
ifile.close()
ofile.close()
if action.hash != fhash:
s = os.stat(filepath)
os.remove(filepath)
raise tx.InvalidContentException(
action.path, "hash failure: expected: %s" "computed: %s" % (action.hash, fhash), size=s.st_size
)
return
def install(self, pkgplan, orig):
"""Client-side method that installs the license."""
mode = 0444
owner = 0
group = 0
path = self.attrs["path"]
stream = self.data()
path = os.path.normpath(os.path.sep.join(
(pkgplan.image.get_root(), path)))
# make sure the directory exists and the file is writable
if not os.path.exists(os.path.dirname(path)):
self.makedirs(os.path.dirname(path), mode=0755)
elif os.path.exists(path):
os.chmod(path, 0644)
lfile = file(path, "wb")
# XXX Should throw an exception if shasum doesn't match
# self.hash
shasum = misc.gunzip_from_stream(stream, lfile)
lfile.close()
stream.close()
os.chmod(path, mode)
try:
portable.chown(path, owner, group)
except OSError, e:
if e.errno != errno.EPERM:
raise
def install(self, pkgplan, orig):
"""Client-side method that installs the license."""
owner = 0
group = 0
path = self.attrs["path"]
stream = self.data()
path = os.path.normpath(
os.path.sep.join((pkgplan.image.get_root(), path)))
# make sure the directory exists and the file is writable
if not os.path.exists(os.path.dirname(path)):
self.makedirs(os.path.dirname(path),
mode=misc.PKG_DIR_MODE,
fmri=pkgplan.destination_fmri)
elif os.path.exists(path):
os.chmod(path, misc.PKG_FILE_MODE)
lfile = file(path, "wb")
# XXX Should throw an exception if shasum doesn't match
# self.hash
shasum = misc.gunzip_from_stream(stream, lfile)
lfile.close()
stream.close()
os.chmod(path, misc.PKG_RO_FILE_MODE)
try:
portable.chown(path, owner, group)
except OSError, e:
if e.errno != errno.EPERM:
raise
def install(self, pkgplan, orig):
"""Client-side method that installs the license."""
owner = 0
group = 0
# ensure "path" is initialized. it may not be if we've loaded
# a plan that was previously prepared.
self.preinstall(pkgplan, orig)
path = self.attrs["path"]
stream = self.data()
path = os.path.normpath(os.path.sep.join(
(pkgplan.image.get_root(), path)))
# make sure the directory exists and the file is writable
if not os.path.exists(os.path.dirname(path)):
self.makedirs(os.path.dirname(path),
mode=misc.PKG_DIR_MODE,
fmri=pkgplan.destination_fmri)
elif os.path.exists(path):
os.chmod(path, misc.PKG_FILE_MODE)
lfile = file(path, "wb")
try:
shasum = misc.gunzip_from_stream(stream, lfile)
except zlib.error, e:
raise ActionExecutionError(self, details=_("Error "
"decompressing payload: %s") %
(" ".join([str(a) for a in e.args])), error=e)
def install(self, pkgplan, orig):
"""Client-side method that installs the license."""
owner = 0
group = 0
# ensure "path" is initialized. it may not be if we've loaded
# a plan that was previously prepared.
self.preinstall(pkgplan, orig)
stream = self.data()
path = self.get_installed_path(pkgplan.image.get_root())
# make sure the directory exists and the file is writable
if not os.path.exists(os.path.dirname(path)):
self.makedirs(os.path.dirname(path),
mode=misc.PKG_DIR_MODE,
fmri=pkgplan.destination_fmri)
elif os.path.exists(path):
os.chmod(path, misc.PKG_FILE_MODE)
lfile = open(path, "wb")
try:
hash_attr, hash_val, hash_func = \
digest.get_preferred_hash(self)
shasum = misc.gunzip_from_stream(stream,
lfile,
hash_func=hash_func)
except zlib.error as e:
raise ActionExecutionError(
self,
details=_("Error "
"decompressing payload: {0}").format(" ".join(
[str(a) for a in e.args])),
error=e)
finally:
lfile.close()
stream.close()
if shasum != hash_val:
raise ActionExecutionError(
self,
details=_("Action "
"data hash verification failure: expected: "
"{expected} computed: {actual} action: "
"{action}").format(expected=hash_val,
actual=shasum,
action=self))
os.chmod(path, misc.PKG_RO_FILE_MODE)
try:
portable.chown(path, owner, group)
except OSError as e:
if e.errno != errno.EPERM:
raise
def _verify_content(action, filepath):
"""If action contains an attribute that has the compressed
hash, read the file specified in filepath and verify
that the hash values match. If the values do not match,
remove the file and raise an InvalidContentException."""
chash = action.attrs.get("chash", None)
path = action.attrs.get("path", None)
if not chash:
# Compressed hash doesn't exist. Decompress and
# generate hash of uncompressed content.
ifile = open(filepath, "rb")
ofile = open(os.devnull, "wb")
try:
fhash = misc.gunzip_from_stream(ifile, ofile)
except zlib.error, e:
s = os.stat(filepath)
os.remove(filepath)
raise tx.InvalidContentException(
path,
"zlib.error:%s" % (" ".join([str(a) for a in e.args])),
size=s.st_size)
ifile.close()
ofile.close()
if action.hash != fhash:
s = os.stat(filepath)
os.remove(filepath)
raise tx.InvalidContentException(action.path,
"hash failure: expected: %s"
"computed: %s" %
(action.hash, fhash),
size=s.st_size)
return
def install(self, pkgplan, orig):
"""Client-side method that installs a file."""
mode = None
try:
mode = int(self.attrs.get("mode", None), 8)
except (TypeError, ValueError):
# Mode isn't valid, so let validate raise a more
# informative error.
self.validate(fmri=pkgplan.destination_fmri)
owner, group = self.get_fsobj_uid_gid(pkgplan,
pkgplan.destination_fmri)
final_path = self.get_installed_path(pkgplan.image.get_root())
# Don't allow installation through symlinks.
self.fsobj_checkpath(pkgplan, final_path)
if not os.path.exists(os.path.dirname(final_path)):
self.makedirs(os.path.dirname(final_path),
mode=misc.PKG_DIR_MODE,
fmri=pkgplan.destination_fmri)
elif (not orig and not pkgplan.origin_fmri and "preserve" in self.attrs
and self.attrs["preserve"] not in ("abandon", "install-only")
and os.path.isfile(final_path)):
# Unpackaged editable file is already present during
# initial install; salvage it before continuing.
pkgplan.salvage(final_path)
# XXX If we're upgrading, do we need to preserve file perms from
# existing file?
# check if we have a save_file active; if so, simulate file
# being already present rather than installed from scratch
if "save_file" in self.attrs:
orig = self.restore_file(pkgplan.image)
# See if we need to preserve the file, and if so, set that up.
#
# XXX What happens when we transition from preserve to
# non-preserve or vice versa? Do we want to treat a preserve
# attribute as turning the action into a critical action?
#
# XXX We should save the originally installed file. It can be
# used as an ancestor for a three-way merge, for example. Where
# should it be stored?
pres_type = self._check_preserve(orig, pkgplan)
do_content = True
old_path = None
if pres_type == True or (pres_type and pkgplan.origin_fmri
== pkgplan.destination_fmri):
# File is marked to be preserved and exists so don't
# reinstall content.
do_content = False
elif pres_type == "legacy":
# Only rename old file if this is a transition to
# preserve=legacy from something else.
if orig.attrs.get("preserve", None) != "legacy":
old_path = final_path + ".legacy"
elif pres_type == "renameold.update":
old_path = final_path + ".update"
elif pres_type == "renameold":
old_path = final_path + ".old"
elif pres_type == "renamenew":
final_path = final_path + ".new"
elif pres_type == "abandon":
return
# If it is a directory (and not empty) then we should
# salvage the contents.
if os.path.exists(final_path) and \
not os.path.islink(final_path) and \
os.path.isdir(final_path):
try:
os.rmdir(final_path)
except OSError as e:
if e.errno == errno.ENOENT:
pass
elif e.errno in (errno.EEXIST, errno.ENOTEMPTY):
pkgplan.salvage(final_path)
elif e.errno != errno.EACCES:
# this happens on Windows
raise
# XXX This needs to be modularized.
if do_content and self.needsdata(orig, pkgplan):
tfilefd, temp = tempfile.mkstemp(dir=os.path.dirname(final_path))
if not self.data:
# The state of the filesystem changed after the
# plan was prepared; attempt a one-off
# retrieval of the data.
self.data = self.__set_data(pkgplan)
stream = self.data()
tfile = os.fdopen(tfilefd, "wb")
try:
# Always verify using the most preferred hash
hash_attr, hash_val, hash_func = \
digest.get_preferred_hash(self)
shasum = misc.gunzip_from_stream(stream, tfile, hash_func)
except zlib.error as e:
raise ActionExecutionError(
self,
details=_("Error decompressing payload: "
"{0}").format(" ".join([str(a)
for a in e.args])),
error=e)
finally:
tfile.close()
stream.close()
if shasum != hash_val:
raise ActionExecutionError(
self,
details=_("Action data hash verification "
"failure: expected: {expected} computed: "
"{actual} action: {action}").format(
expected=hash_val,
actual=shasum,
action=self))
else:
temp = final_path
try:
os.chmod(temp, mode)
except OSError as e:
# If the file didn't exist, assume that's intentional,
# and drive on.
if e.errno != errno.ENOENT:
raise
else:
return
try:
portable.chown(temp, owner, group)
except OSError as e:
if e.errno != errno.EPERM:
raise
# XXX There's a window where final_path doesn't exist, but we
# probably don't care.
if do_content and old_path:
try:
portable.rename(final_path, old_path)
except OSError as e:
if e.errno != errno.ENOENT:
# Only care if file isn't gone already.
raise
# This is safe even if temp == final_path.
try:
portable.rename(temp, final_path)
except OSError as e:
raise api_errors.FileInUseException(final_path)
# Handle timestamp if specified (and content was installed).
if do_content and "timestamp" in self.attrs:
t = misc.timestamp_to_time(self.attrs["timestamp"])
try:
os.utime(final_path, (t, t))
except OSError as e:
if e.errno != errno.EACCES:
raise
# On Windows, the time cannot be changed on a
# read-only file
os.chmod(final_path, stat.S_IRUSR | stat.S_IWUSR)
os.utime(final_path, (t, t))
os.chmod(final_path, mode)
# Handle system attributes.
sattr = self.attrs.get("sysattr")
if sattr:
if isinstance(sattr, list):
sattr = ",".join(sattr)
sattrs = sattr.split(",")
if len(sattrs) == 1 and \
sattrs[0] not in portable.get_sysattr_dict():
# not a verbose attr, try as a compact attr seq
arg = sattrs[0]
else:
arg = sattrs
try:
portable.fsetattr(final_path, arg)
except OSError as e:
if e.errno != errno.EINVAL:
raise
warn = _("System attributes are not supported "
"on the target image filesystem; 'sysattr'"
" ignored for {0}").format(self.attrs["path"])
pkgplan.image.imageplan.pd.add_item_message(
pkgplan.destination_fmri,
misc.time_to_timestamp(time.time()), MSG_WARNING, warn)
except ValueError as e:
warn = _("Could not set system attributes for {path}"
"'{attrlist}': {err}").format(attrlist=sattr,
err=e,
path=self.attrs["path"])
pkgplan.image.imageplan.pd.add_item_message(
pkgplan.destination_fmri,
misc.time_to_timestamp(time.time()), MSG_WARNING, warn)
def add_manifest(self, f):
"""Adds the manifest to the Transaction."""
if isinstance(f, six.string_types):
f = open(f, "rb")
# Store the manifest file.
fpath = os.path.join(self.dir, "manifest")
with open(fpath, "ab+") as wf:
try:
misc.gunzip_from_stream(f, wf, ignore_hash=True)
wf.seek(0)
content = wf.read()
except zlib.error:
# No need to decompress it if it's not a gzipped
# file.
f.seek(0)
content = f.read()
wf.write(content)
# Do some sanity checking on packages marked or being marked
# obsolete or renamed.
m = pkg.manifest.Manifest()
m.set_content(misc.force_str(content))
for action in m.gen_actions():
if action.name == "set" and \
action.attrs["name"] == "pkg.obsolete" and \
action.attrs["value"] == "true":
self.obsolete = True
if self.types_found.difference(
set(("set", "signature"))):
raise TransactionOperationError(_("An obsolete "
"package cannot contain actions other than "
"'set' and 'signature'."))
elif action.name == "set" and \
action.attrs["name"] == "pkg.renamed" and \
action.attrs["value"] == "true":
self.renamed = True
if self.types_found.difference(
set(("depend", "set", "signature"))):
raise TransactionOperationError(_("A renamed "
"package cannot contain actions other than "
"'set', 'depend', and 'signature'."))
if not self.has_reqdeps and action.name == "depend" and \
action.attrs["type"] == "require":
self.has_reqdeps = True
if self.obsolete and self.renamed:
# Reset either obsolete or renamed, depending on which
# action this was.
if action.attrs["name"] == "pkg.obsolete":
self.obsolete = False
else:
self.renamed = False
raise TransactionOperationError(_("A package may not "
" be marked for both obsoletion and renaming."))
elif self.obsolete and action.name not in ("set", "signature"):
raise TransactionOperationError(_("A '{type}' action "
"cannot be present in an obsolete package: "
"{action}").format(
type=action.name, action=action))
elif self.renamed and action.name not in \
("depend", "set", "signature"):
raise TransactionOperationError(_("A '{type}' action "
"cannot be present in a renamed package: "
"{action}").format(
type=action.name, action=action))
self.types_found.add(action.name)
pass
elif e.errno in (errno.EEXIST, errno.ENOTEMPTY):
pkgplan.salvage(final_path)
elif e.errno != errno.EACCES:
# this happens on Windows
raise
# XXX This needs to be modularized.
# XXX This needs to be controlled by policy.
if do_content and self.needsdata(orig, pkgplan):
tfilefd, temp = tempfile.mkstemp(dir=os.path.dirname(
final_path))
stream = self.data()
tfile = os.fdopen(tfilefd, "wb")
try:
shasum = misc.gunzip_from_stream(stream, tfile)
except zlib.error, e:
raise ActionExecutionError(self,
details=_("Error decompressing payload: %s")
% (" ".join([str(a) for a in e.args])),
error=e)
finally:
tfile.close()
stream.close()
if shasum != self.hash:
raise ActionExecutionError(self,
details=_("Action data hash verification "
"failure: expected: %(expected)s computed: "
"%(actual)s action: %(action)s") % {
"expected": self.hash,
except OSError, e:
if e.errno == errno.ENOENT:
pass
elif e.errno in (errno.EEXIST, errno.ENOTEMPTY):
pkgplan.image.salvage(final_path)
elif e.errno != errno.EACCES:
# this happens on Windows
raise
# XXX This needs to be modularized.
# XXX This needs to be controlled by policy.
if do_content and self.needsdata(orig, pkgplan):
tfilefd, temp = tempfile.mkstemp(dir=os.path.dirname(final_path))
stream = self.data()
tfile = os.fdopen(tfilefd, "wb")
shasum = misc.gunzip_from_stream(stream, tfile)
tfile.close()
stream.close()
# XXX Should throw an exception if shasum doesn't match
# self.hash
else:
temp = final_path
try:
os.chmod(temp, mode)
except OSError, e:
# If the file didn't exist, assume that's intentional,
# and drive on.
if e.errno != errno.ENOENT:
data=req_str)[0]
except:
abort(err=_("Unable to retrieve content from: %s") % src_uri)
tar_stream = ptf.PkgTarFile.open(mode = "r|", fileobj = f)
for info in tar_stream:
gzfobj = None
try:
if not keep_compressed:
# Uncompress as we retrieve the files
gzfobj = tar_stream.extractfile(info)
fpath = os.path.join(tmpdir,
info.name)
outfile = open(fpath, "wb")
gunzip_from_stream(gzfobj, outfile)
outfile.close()
gzfobj.close()
else:
# We want to keep the files compressed
# on disk.
tar_stream.extract_to(info, tmpdir,
info.name)
# Copy the file into place (rename can cause a cross-
# link device failure) and then remove the original.
src = os.path.join(tmpdir, info.name)
shutil.copy(src, os.path.join(destdir, info.name))
portable.remove(src)
tracker.download_add_progress(1, cshashes[info.name][2])
try:
f = versioned_urlopen(src_uri, "filelist", [0], data=req_str)[0]
except:
abort(err=_("Unable to retrieve content from: %s") % src_uri)
tar_stream = ptf.PkgTarFile.open(mode="r|", fileobj=f)
for info in tar_stream:
gzfobj = None
try:
if not keep_compressed:
# Uncompress as we retrieve the files
gzfobj = tar_stream.extractfile(info)
fpath = os.path.join(tmpdir, info.name)
outfile = open(fpath, "wb")
gunzip_from_stream(gzfobj, outfile)
outfile.close()
gzfobj.close()
else:
# We want to keep the files compressed
# on disk.
tar_stream.extract_to(info, tmpdir, info.name)
# Copy the file into place (rename can cause a cross-
# link device failure) and then remove the original.
src = os.path.join(tmpdir, info.name)
shutil.copy(src, os.path.join(destdir, info.name))
portable.remove(src)
tracker.download_add_progress(1, cshashes[info.name][2])
except KeyboardInterrupt: