def __set_chain_certs_data(self, chain_certs, chash_dir):
"""Store the information about the certs needed to validate
this signature in the signature.
The 'chain_certs' parameter is a list of paths to certificates.
"""
self.chain_cert_openers = []
hshes = []
sizes = []
chshes = []
csizes = []
for pth in chain_certs:
if not os.path.exists(pth):
raise pkg.actions.ActionDataError(
_("No such file: '%s'.") % pth, path=pth)
elif os.path.isdir(pth):
raise pkg.actions.ActionDataError(
_("'%s' is not a file.") % pth, path=pth)
file_opener = self.make_opener(pth)
self.chain_cert_openers.append(file_opener)
self.attrs.setdefault("chain.sizes", [])
try:
fs = os.stat(pth)
sizes.append(str(fs.st_size))
except EnvironmentError, e:
raise pkg.actions.ActionDataError(e, path=pth)
# misc.get_data_digest takes care of closing the file
# that's opened below.
with file_opener() as fh:
hsh, data = misc.get_data_digest(fh,
length=fs.st_size, return_content=True)
hshes.append(hsh)
csize, chash = misc.compute_compressed_attrs(hsh,
None, data, fs.st_size, chash_dir)
csizes.append(csize)
chshes.append(chash.hexdigest())
def add_file(self, f, size=None):
"""Adds the file to the Transaction."""
hashes, data = misc.get_data_digest(
f,
length=size,
return_content=True,
hash_attrs=digest.DEFAULT_HASH_ATTRS,
hash_algs=digest.HASH_ALGS)
if size is None:
size = len(data)
try:
# We don't have an Action yet, so passing None is fine.
default_hash_attr = digest.get_least_preferred_hash(None)[0]
fname = hashes[default_hash_attr]
dst_path = self.rstore.file(fname)
except Exception as e:
# The specific exception can't be named here due
# to the cyclic dependency between this class
# and the repository class.
if getattr(e, "data", "") != fname:
raise
dst_path = None
csize, chashes = misc.compute_compressed_attrs(
fname,
dst_path,
data,
size,
self.dir,
chash_attrs=digest.DEFAULT_CHASH_ATTRS,
chash_algs=digest.CHASH_ALGS)
chashes = None
data = None
self.remaining_payload_cnt -= 1
def __set_chain_certs_data(self, chain_certs, chash_dir):
"""Store the information about the certs needed to validate
this signature in the signature.
The 'chain_certs' parameter is a list of paths to certificates.
"""
self.chain_cert_openers = []
# chain_hshes and chain_chshes are dictionaries which map a
# given hash or compressed hash attribute to a list of the hash
# values for each path in chain_certs.
chain_hshes = {}
chain_chshes = {}
chain_csizes = []
chain_sizes = []
for attr in digest.DEFAULT_CHAIN_ATTRS:
chain_hshes[attr] = []
for attr in digest.DEFAULT_CHAIN_CHASH_ATTRS:
chain_chshes[attr] = []
for pth in chain_certs:
if not os.path.exists(pth):
raise pkg.actions.ActionDataError(
_("No such file: '{0}'.").format(pth), path=pth)
elif os.path.isdir(pth):
raise pkg.actions.ActionDataError(
_("'{0}' is not a file.").format(pth), path=pth)
file_opener = self.make_opener(pth)
self.chain_cert_openers.append(file_opener)
self.attrs.setdefault("chain.sizes", [])
self.attrs.setdefault("chain.csizes", [])
try:
fs = os.stat(pth)
chain_sizes.append(str(fs.st_size))
except EnvironmentError as e:
raise pkg.actions.ActionDataError(e, path=pth)
# misc.get_data_digest takes care of closing the file
# that's opened below.
with file_opener() as fh:
hshes, data = misc.get_data_digest(
fh,
length=fs.st_size,
return_content=True,
hash_attrs=digest.DEFAULT_CHAIN_ATTRS,
hash_algs=digest.CHAIN_ALGS)
for attr in hshes:
chain_hshes[attr].append(hshes[attr])
# We need a filename to use for the uncompressed chain
# cert, so get the preferred chain hash value from the
# chain_hshes
alg = digest.PREFERRED_HASH
if alg == "sha1":
attr = "chain"
else:
attr = "pkg.chain.{0}".format(alg)
chain_val = hshes.get(attr)
csize, chashes = misc.compute_compressed_attrs(
chain_val,
None,
data,
fs.st_size,
chash_dir,
chash_attrs=digest.DEFAULT_CHAIN_CHASH_ATTRS,
chash_algs=digest.CHAIN_CHASH_ALGS)
chain_csizes.append(csize)
for attr in chashes:
chain_chshes[attr].append(chashes[attr])
# Remove any unused hash attributes.
for cattrs in (chain_hshes, chain_chshes):
for attr in list(cattrs.keys()):
if not cattrs[attr]:
cattrs.pop(attr, None)
if chain_hshes:
# These attributes are stored as a single value with
# spaces in it rather than multiple values to ensure
# the ordering remains consistent.
self.attrs["chain.sizes"] = " ".join(chain_sizes)
self.attrs["chain.csizes"] = " ".join(chain_csizes)
for attr in digest.DEFAULT_CHAIN_ATTRS:
self.attrs[attr] = " ".join(chain_hshes[attr])
for attr in digest.DEFAULT_CHAIN_CHASH_ATTRS:
self.attrs[attr] = " ".join(chain_chshes[attr])
def sig_str(self, a, version):
"""Create a stable string representation of an action that
is deterministic in its creation. If creating a string from an
action is non-deterministic, then manifest signing cannot work.
The parameter 'a' is the signature action that's going to use
the string produced. It's needed for the signature string
action, and is here to keep the method signature the same.
"""
# Any changes to this function mean Action.sig_version must be
# incremented.
if version != generic.Action.sig_version:
raise apx.UnsupportedSignatureVersion(version, sig=self)
# Signature actions don't sign other signature actions. So if
# the action that's doing the signing isn't ourself, return
# nothing.
if str(a) != str(self):
return None
# It's necessary to sign the action as the client will see it,
# post publication. To do that, it's necessary to simulate the
# publication process on a copy of the action, converting
# paths to hashes and adding size information.
tmp_a = SignatureAction(None, **self.attrs)
# The signature action can't sign the value of the value
# attribute, but it can sign that attribute's name.
tmp_a.attrs["value"] = ""
if hasattr(self.data, "__call__"):
size = int(self.attrs.get("pkg.size", 0))
tmp_dir = tempfile.mkdtemp()
with self.data() as fh:
hashes, data = misc.get_data_digest(
fh,
size,
return_content=True,
hash_attrs=digest.DEFAULT_HASH_ATTRS,
hash_algs=digest.HASH_ALGS)
tmp_a.attrs.update(hashes)
# "hash" is special since it shouldn't appear in
# the action attributes, it gets set as a member
# instead.
if "hash" in tmp_a.attrs:
tmp_a.hash = tmp_a.attrs["hash"]
del tmp_a.attrs["hash"]
# The use of self.hash here is just to point to a
# filename, the type of hash used for self.hash is
# irrelevant. Note that our use of self.hash for the
# basename will need to be modified when we finally move
# off SHA-1 hashes.
csize, chashes = misc.compute_compressed_attrs(
os.path.basename(self.hash), self.hash, data, size, tmp_dir)
shutil.rmtree(tmp_dir)
tmp_a.attrs["pkg.csize"] = csize
for attr in chashes:
tmp_a.attrs[attr] = chashes[attr]
elif self.hash:
tmp_a.hash = self.hash
for attr in digest.DEFAULT_HASH_ATTRS:
if attr in self.attrs:
tmp_a.attrs[attr] = self.attrs[attr]
csizes = []
chain_hashes = {}
chain_chashes = {}
for attr in digest.DEFAULT_CHAIN_ATTRS:
chain_hashes[attr] = []
for attr in digest.DEFAULT_CHAIN_CHASH_ATTRS:
chain_chashes[attr] = []
sizes = self.attrs.get("chain.sizes", "").split()
for i, c in enumerate(self.chain_cert_openers):
size = int(sizes[i])
tmp_dir = tempfile.mkdtemp()
hshes, data = misc.get_data_digest(
c(),
size,
return_content=True,
hash_attrs=digest.DEFAULT_CHAIN_ATTRS,
hash_algs=digest.CHAIN_ALGS)
for attr in hshes:
chain_hashes[attr].append(hshes[attr])
csize, chashes = misc.compute_compressed_attrs(
"tmp",
None,
data,
size,
tmp_dir,
chash_attrs=digest.DEFAULT_CHAIN_CHASH_ATTRS,
chash_algs=digest.CHAIN_CHASH_ALGS)
shutil.rmtree(tmp_dir)
csizes.append(csize)
for attr in chashes:
chain_chashes[attr].append(chashes[attr])
if chain_hashes:
for attr in digest.DEFAULT_CHAIN_ATTRS:
if chain_hashes[attr]:
tmp_a.attrs[attr] = " ".join(chain_hashes[attr])
# Now that tmp_a looks like the post-published action, transform
# it into a string using the generic sig_str method.
return generic.Action.sig_str(tmp_a, tmp_a, version)
def _process_action(self, action, exact=False, path=None):
"""Adds all expected attributes to the provided action and
upload the file for the action if needed.
If 'exact' is True and 'path' is 'None', the action won't
be modified and no file will be uploaded.
If 'exact' is True and a 'path' is provided, the file of that
path will be uploaded as-is (it is assumed that the file is
already in repository format).
"""
if self._append_mode and action.name != "signature":
raise TransactionOperationError(non_sig=True)
size = int(action.attrs.get("pkg.size", 0))
if action.has_payload and size <= 0:
# XXX hack for empty files
action.data = lambda: open(os.devnull, "rb")
if action.data is None:
return
if exact:
if path:
self.add_file(path,
basename=action.hash,
progtrack=self.progtrack)
return
# Get all hashes for this action.
hashes, dummy = misc.get_data_digest(
action.data(),
length=size,
hash_attrs=digest.DEFAULT_HASH_ATTRS,
hash_algs=digest.HASH_ALGS)
# Set the hash member for backwards compatibility and
# remove it from the dictionary.
action.hash = hashes.pop("hash", None)
action.attrs.update(hashes)
# Add file content-hash when preferred_hash is SHA2 or higher.
if action.name != "signature" and \
digest.PREFERRED_HASH != "sha1":
hash_attr = "{0}:{1}".format(digest.EXTRACT_FILE,
digest.PREFERRED_HASH)
file_content_hash, dummy = misc.get_data_digest(
action.data(),
length=size,
return_content=False,
hash_attrs=[hash_attr],
hash_algs=digest.HASH_ALGS)
action.attrs["pkg.content-hash"] = "{0}:{1}".format(
hash_attr, file_content_hash[hash_attr])
# Now set the hash value that will be used for storing the file
# in the repository.
hash_attr, hash_val, hash_func = \
digest.get_least_preferred_hash(action)
fname = hash_val
hdata = self.__uploads.get(fname)
if hdata is not None:
elf_attrs, csize, chashes = hdata
else:
# We haven't processed this file before, determine if
# it needs to be uploaded and what information the
# repository knows about it.
elf_attrs = self.__get_elf_attrs(action, fname, size)
csize, chashes = self.__get_compressed_attrs(fname)
# 'csize' indicates that if file needs to be uploaded.
fileneeded = csize is None
if fileneeded:
fpath = os.path.join(self._tmpdir, fname)
csize, chashes = misc.compute_compressed_attrs(
fname,
data=action.data(),
size=size,
compress_dir=self._tmpdir)
# Upload the compressed file for each action.
self.add_file(fpath, basename=fname, progtrack=self.progtrack)
os.unlink(fpath)
self.__uploaded += 1
elif not chashes:
# If not fileneeded, and repository can't
# provide desired hashes, call
# compute_compressed_attrs() in a way that
# avoids writing the file to get the attributes
# we need.
csize, chashes = misc.compute_compressed_attrs(
fname, data=action.data(), size=size)
self.__uploads[fname] = (elf_attrs, csize, chashes)
for k, v in six.iteritems(elf_attrs):
if isinstance(v, list):
action.attrs[k] = v + action.attrlist(k)
else:
action.attrs[k] = v
for k, v in six.iteritems(chashes):
if k == "pkg.content-hash":
action.attrs[k] = action.attrlist(k) + [v]
else:
action.attrs[k] = v
action.attrs["pkg.csize"] = csize
def add_file(self, f, basename=None, size=None):
"""Adds the file to the Transaction."""
# If basename provided, just store the file as-is with the
# basename.
if basename:
fileneeded = True
try:
dst_path = self.rstore.file(basename)
fileneeded = False
except Exception as e:
dst_path = os.path.join(self.dir, basename)
if not fileneeded:
return
if isinstance(f, six.string_types):
portable.copyfile(f, dst_path)
return
bufsz = 128 * 1024
if bufsz > size:
bufsz = size
with open(dst_path, "wb") as wf:
while True:
data = f.read(bufsz)
# data is bytes
if data == b"":
break
wf.write(data)
return
hashes, data = misc.get_data_digest(f, length=size,
return_content=True, hash_attrs=digest.DEFAULT_HASH_ATTRS,
hash_algs=digest.HASH_ALGS)
if size is None:
size = len(data)
fname = None
try:
# We don't have an Action yet, so passing None is fine.
default_hash_attr = digest.get_least_preferred_hash(
None)[0]
fname = hashes[default_hash_attr]
dst_path = self.rstore.file(fname)
except Exception as e:
# The specific exception can't be named here due
# to the cyclic dependency between this class
# and the repository class.
if getattr(e, "data", "") != fname:
raise
dst_path = None
misc.compute_compressed_attrs(fname, dst_path,
data, size, self.dir,
chash_attrs=digest.DEFAULT_CHASH_ATTRS,
chash_algs=digest.CHASH_ALGS)
self.remaining_payload_cnt -= 1
def add_content(self, action):
"""Adds the content of the provided action (if applicable) to
the Transaction."""
# Perform additional publication-time validation of actions
# before further processing is done.
try:
action.validate()
except actions.ActionError as e:
raise TransactionOperationError(e)
if self.append_trans and action.name != "signature":
raise TransactionOperationError(non_sig=True)
size = int(action.attrs.get("pkg.size", 0))
if action.has_payload and size <= 0:
# XXX hack for empty files
action.data = lambda: open(os.devnull, "rb")
if action.data is not None:
# get all hashes for this action
hashes, data = misc.get_data_digest(action.data(),
length=size, return_content=True,
hash_attrs=digest.LEGACY_HASH_ATTRS,
hash_algs=digest.HASH_ALGS)
# set the hash member for backwards compatibility and
# remove it from the dictionary
action.hash = hashes.pop("hash", None)
action.attrs.update(hashes)
# now set the hash value that will be used for storing
# the file in the repository.
hash_attr, hash_val, hash_func = \
digest.get_least_preferred_hash(action)
fname = hash_val
# Extract ELF information if not already provided.
# XXX This needs to be modularized.
if haveelf and data[:4] == b"\x7fELF" and (
"elfarch" not in action.attrs or
"elfbits" not in action.attrs or
"elfhash" not in action.attrs):
elf_name = os.path.join(self.dir,
".temp-{0}".format(fname))
elf_file = open(elf_name, "wb")
elf_file.write(data)
elf_file.close()
try:
elf_info = elf.get_info(elf_name)
except elf.ElfError as e:
raise TransactionContentError(e)
try:
# Check which content checksums to
# compute and add to the action
elf1 = "elfhash"
if elf1 in \
digest.LEGACY_CONTENT_HASH_ATTRS:
get_sha1 = True
else:
get_sha1 = False
hashes = elf.get_hashes(elf_name,
elfhash=get_sha1)
if get_sha1:
action.attrs[elf1] = hashes[elf1]
except elf.ElfError:
pass
action.attrs["elfbits"] = str(elf_info["bits"])
action.attrs["elfarch"] = elf_info["arch"]
os.unlink(elf_name)
try:
dst_path = self.rstore.file(fname)
except Exception as e:
# The specific exception can't be named here due
# to the cyclic dependency between this class
# and the repository class.
if getattr(e, "data", "") != fname:
raise
dst_path = None
csize, chashes = misc.compute_compressed_attrs(
fname, dst_path, data, size, self.dir)
for attr in chashes:
action.attrs[attr] = chashes[attr]
action.attrs["pkg.csize"] = csize
self.remaining_payload_cnt = \
len(action.attrs.get("chain.sizes", "").split())
# Do some sanity checking on packages marked or being marked
# obsolete or renamed.
if action.name == "set" and \
action.attrs["name"] == "pkg.obsolete" and \
action.attrs["value"] == "true":
self.obsolete = True
if self.types_found.difference(
set(("set", "signature"))):
raise TransactionOperationError(_("An obsolete "
"package cannot contain actions other than "
"'set' and 'signature'."))
elif action.name == "set" and \
action.attrs["name"] == "pkg.renamed" and \
action.attrs["value"] == "true":
self.renamed = True
if self.types_found.difference(
set(("depend", "set", "signature"))):
raise TransactionOperationError(_("A renamed "
"package cannot contain actions other than "
"'set', 'depend', and 'signature'."))
if not self.has_reqdeps and action.name == "depend" and \
action.attrs["type"] == "require":
self.has_reqdeps = True
if self.obsolete and self.renamed:
# Reset either obsolete or renamed, depending on which
# action this was.
if action.attrs["name"] == "pkg.obsolete":
self.obsolete = False
else:
self.renamed = False
raise TransactionOperationError(_("A package may not "
" be marked for both obsoletion and renaming."))
elif self.obsolete and action.name not in ("set", "signature"):
raise TransactionOperationError(_("A '{type}' action "
"cannot be present in an obsolete package: "
"{action}").format(
type=action.name, action=action))
elif self.renamed and action.name not in \
("depend", "set", "signature"):
raise TransactionOperationError(_("A '{type}' action "
"cannot be present in a renamed package: "
"{action}").format(
type=action.name, action=action))
# Now that the action is known to be sane, we can add it to the
# manifest.
tfpath = os.path.join(self.dir, "manifest")
tfile = open(tfpath, "a+")
print(action, file=tfile)
tfile.close()
self.types_found.add(action.name)
def sig_str(self, a, version):
"""Create a stable string representation of an action that
is deterministic in its creation. If creating a string from an
action is non-deterministic, then manifest signing cannot work.
The parameter 'a' is the signature action that's going to use
the string produced. It's needed for the signature string
action, and is here to keep the method signature the same.
"""
# Any changes to this function mean Action.sig_version must be
# incremented.
if version != generic.Action.sig_version:
raise apx.UnsupportedSignatureVersion(version, sig=self)
# Signature actions don't sign other signature actions. So if
# the action that's doing the signing isn't ourself, return
# nothing.
if str(a) != str(self):
return None
# It's necessary to sign the action as the client will see it,
# post publication. To do that, it's necessary to simulate the
# publication process on a copy of the action, converting
# paths to hashes and adding size information.
tmp_a = SignatureAction(None, **self.attrs)
# The signature action can't sign the value of the value
# attribute, but it can sign that attribute's name.
tmp_a.attrs["value"] = ""
if callable(self.data):
size = int(self.attrs.get("pkg.size", 0))
tmp_dir = tempfile.mkdtemp()
with self.data() as fh:
tmp_a.hash, data = misc.get_data_digest(fh,
size, return_content=True)
csize, chash = misc.compute_compressed_attrs(
os.path.basename(self.hash), self.hash, data, size,
tmp_dir)
shutil.rmtree(tmp_dir)
tmp_a.attrs["pkg.csize"] = csize
tmp_a.attrs["chash"] = chash.hexdigest()
elif self.hash:
tmp_a.hash = self.hash
hashes = []
csizes = []
chashes = []
sizes = self.attrs.get("chain.sizes", "").split()
for i, c in enumerate(self.chain_cert_openers):
size = int(sizes[i])
tmp_dir = tempfile.mkdtemp()
hsh, data = misc.get_data_digest(c(), size,
return_content=True)
hashes.append(hsh)
csize, chash = misc.compute_compressed_attrs("tmp",
None, data, size, tmp_dir)
shutil.rmtree(tmp_dir)
csizes.append(csize)
chashes.append(chash.hexdigest())
if hashes:
tmp_a.attrs["chain"] = " ".join(hashes)
# Now that tmp_a looks like the post-published action, transform
# it into a string using the generic sig_str method.
return generic.Action.sig_str(tmp_a, tmp_a, version)
pass
action.attrs["elfbits"] = str(elf_info["bits"])
action.attrs["elfarch"] = elf_info["arch"]
os.unlink(elf_name)
try:
dst_path = self.rstore.file(fname)
except Exception, e:
# The specific exception can't be named here due
# to the cyclic dependency between this class
# and the repository class.
if getattr(e, "data", "") != fname:
raise
dst_path = None
csize, chash = misc.compute_compressed_attrs(
fname, dst_path, data, size, self.dir)
action.attrs["chash"] = chash.hexdigest()
action.attrs["pkg.csize"] = csize
chash = None
data = None
self.remaining_payload_cnt = \
len(action.attrs.get("chain.sizes", "").split())
# Do some sanity checking on packages marked or being marked
# obsolete or renamed.
if action.name == "set" and \
action.attrs["name"] == "pkg.obsolete" and \
action.attrs["value"] == "true":
self.obsolete = True
if self.types_found.difference(
