def _WriteExtents(file_obj, data, extents, block_size, base_name):
"""Writes data to file as defined by extent sequence.
This tries to be efficient by not copy data as it is written in chunks.
Args:
file_obj: file object
data: data to write
extents: sequence of block extents (offset and length)
block_size: size of each block
base_name: name string of extent sequence for error reporting
Raises:
PayloadError when things don't add up.
"""
data_offset = 0
data_length = len(data)
for ex, ex_name in common.ExtentIter(extents, base_name):
if not data_length:
raise PayloadError('%s: more write extents than data' % ex_name)
write_length = min(data_length, ex.num_blocks * block_size)
# Only do actual writing if this is not a pseudo-extent.
if ex.start_block != common.PSEUDO_EXTENT_MARKER:
file_obj.seek(ex.start_block * block_size)
data_view = buffer(data, data_offset, write_length)
file_obj.write(data_view)
data_offset += write_length
data_length -= write_length
if data_length:
raise PayloadError('%s: more data than write extents' % base_name)
def Run(self,
new_kernel_part,
new_rootfs_part,
old_kernel_part=None,
old_rootfs_part=None):
"""Applier entry point, invoking all update operations.
Args:
new_kernel_part: name of dest kernel partition file
new_rootfs_part: name of dest rootfs partition file
old_kernel_part: name of source kernel partition file (optional)
old_rootfs_part: name of source rootfs partition file (optional)
Raises:
PayloadError if payload application failed.
"""
self.payload.ResetFile()
# Make sure the arguments are sane and match the payload.
if not (new_kernel_part and new_rootfs_part):
raise PayloadError('missing dst {kernel,rootfs} partitions')
if not (old_kernel_part or old_rootfs_part):
if not self.payload.IsFull():
raise PayloadError(
'trying to apply a non-full update without src '
'{kernel,rootfs} partitions')
elif old_kernel_part and old_rootfs_part:
if not self.payload.IsDelta():
raise PayloadError(
'trying to apply a non-delta update onto src '
'{kernel,rootfs} partitions')
else:
raise PayloadError('not all src partitions provided')
# Apply update to rootfs.
self._ApplyToPartition(self.payload.manifest.install_operations,
'rootfs', 'install_operations', new_rootfs_part,
self.payload.manifest.new_rootfs_info,
old_rootfs_part,
self.payload.manifest.old_rootfs_info)
# Apply update to kernel update.
self._ApplyToPartition(self.payload.manifest.kernel_install_operations,
'kernel', 'kernel_install_operations',
new_kernel_part,
self.payload.manifest.new_kernel_info,
old_kernel_part,
self.payload.manifest.old_kernel_info)
def Read(file_obj, length, offset=None, hasher=None):
"""Reads binary data from a file.
Args:
file_obj: an open file object
length: the length of the data to read
offset: an offset to seek to prior to reading; this is an absolute offset
from either the beginning (non-negative) or end (negative) of the
file. (optional)
hasher: a hashing object to pass the read data through (optional)
Returns:
A string containing the read data.
Raises:
PayloadError if a read error occurred or not enough data was read.
"""
if offset is not None:
if offset >= 0:
file_obj.seek(offset)
else:
file_obj.seek(offset, 2)
try:
data = file_obj.read(length)
except IOError, e:
raise PayloadError('error reading from file (%s): %s' %
(file_obj.name, e))
def Init(self):
"""Initializes the payload object.
This is a prerequisite for any other public API call.
Raises:
PayloadError if object already initialized or fails to initialize
correctly.
"""
if self.is_init:
raise PayloadError('payload object already initialized')
# Initialize hash context.
# pylint: disable=E1101
self.manifest_hasher = hashlib.sha256()
# Read the file header.
self.header = self._ReadHeader()
# Read the manifest.
manifest_raw = self._ReadManifest()
self.manifest = update_metadata_pb2.DeltaArchiveManifest()
self.manifest.ParseFromString(manifest_raw)
# Read the metadata signature (if any).
metadata_signature_raw = self._ReadMetadataSignature()
if metadata_signature_raw:
self.metadata_signature = update_metadata_pb2.Signatures()
self.metadata_signature.ParseFromString(metadata_signature_raw)
self.metadata_size = self.header.size + self.header.manifest_len
self.data_offset = self.metadata_size + self.header.metadata_signature_len
self.is_init = True
def _ApplySourceCopyOperation(self, op, op_name, old_part_file,
new_part_file):
"""Applies a SOURCE_COPY operation.
Args:
op: the operation object
op_name: name string for error reporting
old_part_file: the old partition file object
new_part_file: the new partition file object
Raises:
PayloadError if something goes wrong.
"""
if not old_part_file:
raise PayloadError(
'%s: no source partition file provided for operation type (%d)'
% (op_name, op.type))
block_size = self.block_size
# Gather input raw data from src extents.
in_data = _ReadExtents(old_part_file, op.src_extents, block_size)
# Dump extracted data to dst extents.
_WriteExtents(new_part_file, in_data, op.dst_extents, block_size,
'%s.dst_extents' % op_name)
def IntPackingFmtStr(size, is_unsigned):
"""Returns an integer format string for use by the struct module.
Args:
size: the integer size in bytes (2, 4 or 8)
is_unsigned: whether it is signed or not
Returns:
A format string for packing/unpacking integer values; assumes network byte
order (big-endian).
Raises:
PayloadError if something is wrong with the arguments.
"""
# Determine the base conversion format.
if size == 2:
fmt = 'h'
elif size == 4:
fmt = 'i'
elif size == 8:
fmt = 'q'
else:
raise PayloadError('unsupport numeric field size (%s)' % size)
# Signed or unsigned?
if is_unsigned:
fmt = fmt.upper()
# Make it network byte order (big-endian).
fmt = '!' + fmt
return fmt
def _ExtentsToBspatchArg(extents, block_size, base_name, data_length=-1):
"""Translates an extent sequence into a bspatch-compatible string argument.
Args:
extents: sequence of block extents (offset and length)
block_size: size of each block
base_name: name string of extent sequence for error reporting
data_length: the actual total length of the data in bytes (optional)
Returns:
A tuple consisting of (i) a string of the form
"off_1:len_1,...,off_n:len_n", (ii) an offset where zero padding is needed
for filling the last extent, (iii) the length of the padding (zero means no
padding is needed and the extents cover the full length of data).
Raises:
PayloadError if data_length is too short or too long.
"""
arg = ''
pad_off = pad_len = 0
if data_length < 0:
data_length = sys.maxint
for ex, ex_name in common.ExtentIter(extents, base_name):
if not data_length:
raise PayloadError('%s: more extents than total data length' %
ex_name)
is_pseudo = ex.start_block == common.PSEUDO_EXTENT_MARKER
start_byte = -1 if is_pseudo else ex.start_block * block_size
num_bytes = ex.num_blocks * block_size
if data_length < num_bytes:
# We're only padding a real extent.
if not is_pseudo:
pad_off = start_byte + data_length
pad_len = num_bytes - data_length
num_bytes = data_length
arg += '%s%d:%d' % (arg and ',', start_byte, num_bytes)
data_length -= num_bytes
if data_length:
raise PayloadError('%s: extents not covering full data length' %
base_name)
return arg, pad_off, pad_len
def _VerifySha256(file_obj, expected_hash, name, length=-1):
"""Verifies the SHA256 hash of a file.
Args:
file_obj: file object to read
expected_hash: the hash digest we expect to be getting
name: name string of this hash, for error reporting
length: precise length of data to verify (optional)
Raises:
PayloadError if computed hash doesn't match expected one, or if fails to
read the specified length of data.
"""
# pylint: disable=E1101
hasher = hashlib.sha256()
block_length = 1024 * 1024
max_length = length if length >= 0 else sys.maxint
while max_length > 0:
read_length = min(max_length, block_length)
data = file_obj.read(read_length)
if not data:
break
max_length -= len(data)
hasher.update(data)
if length >= 0 and max_length > 0:
raise PayloadError(
'insufficient data (%d instead of %d) when verifying %s' %
(length - max_length, length, name))
actual_hash = hasher.digest()
if actual_hash != expected_hash:
raise PayloadError('%s hash (%s) not as expected (%s)' %
(name, common.FormatSha256(actual_hash),
common.FormatSha256(expected_hash)))
def _ReadManifest(self):
"""Reads and returns the payload manifest.
Returns:
A string containing the payload manifest in binary form.
Raises:
PayloadError if a read error occurred.
"""
if not self.header:
raise PayloadError('payload header not present')
return common.Read(self.payload_file,
self.header.manifest_len,
hasher=self.manifest_hasher)
def _ReadMetadataSignature(self):
"""Reads and returns the metadata signatures.
Returns:
A string containing the metadata signatures protobuf in binary form or
an empty string if no metadata signature found in the payload.
Raises:
PayloadError if a read error occurred.
"""
if not self.header:
raise PayloadError('payload header not present')
return common.Read(self.payload_file,
self.header.metadata_signature_len,
offset=self.header.size + self.header.manifest_len)
def ReadFromPayload(self, payload_file, hasher=None):
"""Reads the payload header from a file.
Reads the payload header from the |payload_file| and updates the |hasher|
if one is passed. The parsed header is stored in the _PayloadHeader
instance attributes.
Args:
payload_file: a file object
hasher: an optional hasher to pass the value through
Returns:
None.
Raises:
PayloadError if a read error occurred or the header is invalid.
"""
# Verify magic
magic = common.Read(payload_file, len(self._MAGIC), hasher=hasher)
if magic != self._MAGIC:
raise PayloadError('invalid payload magic: %s' % magic)
self.version = _ReadInt(payload_file,
self._VERSION_SIZE,
True,
hasher=hasher)
self.manifest_len = _ReadInt(payload_file,
self._MANIFEST_LEN_SIZE,
True,
hasher=hasher)
self.size = (len(self._MAGIC) + self._VERSION_SIZE +
self._MANIFEST_LEN_SIZE)
self.metadata_signature_len = 0
if self.version == common.BRILLO_MAJOR_PAYLOAD_VERSION:
self.size += self._METADATA_SIGNATURE_LEN_SIZE
self.metadata_signature_len = _ReadInt(
payload_file,
self._METADATA_SIGNATURE_LEN_SIZE,
True,
hasher=hasher)
def _ApplyOperations(self, operations, base_name, old_part_file,
new_part_file, part_size):
"""Applies a sequence of update operations to a partition.
This assumes an in-place update semantics for MOVE and BSDIFF, namely all
reads are performed first, then the data is processed and written back to
the same file.
Args:
operations: the sequence of operations
base_name: the name of the operation sequence
old_part_file: the old partition file object, open for reading/writing
new_part_file: the new partition file object, open for reading/writing
part_size: the partition size
Raises:
PayloadError if anything goes wrong while processing the payload.
"""
for op, op_name in common.OperationIter(operations, base_name):
# Read data blob.
data = self.payload.ReadDataBlob(op.data_offset, op.data_length)
if op.type in (common.OpType.REPLACE, common.OpType.REPLACE_BZ):
self._ApplyReplaceOperation(op, op_name, data, new_part_file,
part_size)
elif op.type == common.OpType.MOVE:
self._ApplyMoveOperation(op, op_name, new_part_file)
elif op.type == common.OpType.BSDIFF:
self._ApplyBsdiffOperation(op, op_name, data, new_part_file)
elif op.type == common.OpType.SOURCE_COPY:
self._ApplySourceCopyOperation(op, op_name, old_part_file,
new_part_file)
elif op.type in (common.OpType.SOURCE_BSDIFF,
common.OpType.IMGDIFF):
self._ApplyDiffOperation(op, op_name, data, old_part_file,
new_part_file)
else:
raise PayloadError('%s: unknown operation type (%d)' %
(op_name, op.type))
def _AssertInit(self):
"""Raises an exception if the object was not initialized."""
if not self.is_init:
raise PayloadError('payload object not initialized')
"""
if offset is not None:
if offset >= 0:
file_obj.seek(offset)
else:
file_obj.seek(offset, 2)
try:
data = file_obj.read(length)
except IOError, e:
raise PayloadError('error reading from file (%s): %s' %
(file_obj.name, e))
if len(data) != length:
raise PayloadError(
'reading from file (%s) too short (%d instead of %d bytes)' %
(file_obj.name, len(data), length))
if hasher:
hasher.update(data)
return data
#
# Formatting functions.
#
def FormatExtent(ex, block_size=0):
end_block = ex.start_block + ex.num_blocks
if block_size:
return '%d->%d * %d' % (ex.start_block, end_block, block_size)
def _ApplyToPartition(self,
operations,
part_name,
base_name,
new_part_file_name,
new_part_info,
old_part_file_name=None,
old_part_info=None):
"""Applies an update to a partition.
Args:
operations: the sequence of update operations to apply
part_name: the name of the partition, for error reporting
base_name: the name of the operation sequence
new_part_file_name: file name to write partition data to
new_part_info: size and expected hash of dest partition
old_part_file_name: file name of source partition (optional)
old_part_info: size and expected hash of source partition (optional)
Raises:
PayloadError if anything goes wrong with the update.
"""
# Do we have a source partition?
if old_part_file_name:
# Verify the source partition.
with open(old_part_file_name, 'rb') as old_part_file:
_VerifySha256(old_part_file,
old_part_info.hash,
'old ' + part_name,
length=old_part_info.size)
new_part_file_mode = 'r+b'
if self.minor_version == common.INPLACE_MINOR_PAYLOAD_VERSION:
# Copy the src partition to the dst one; make sure we don't truncate it.
shutil.copyfile(old_part_file_name, new_part_file_name)
elif (self.minor_version == common.SOURCE_MINOR_PAYLOAD_VERSION or
self.minor_version == common.OPSRCHASH_MINOR_PAYLOAD_VERSION
or self.minor_version
== common.IMGDIFF_MINOR_PAYLOAD_VERSION):
# In minor version >= 2, we don't want to copy the partitions, so
# instead just make the new partition file.
open(new_part_file_name, 'w').close()
else:
raise PayloadError("Unknown minor version: %d" %
self.minor_version)
else:
# We need to create/truncate the dst partition file.
new_part_file_mode = 'w+b'
# Apply operations.
with open(new_part_file_name, new_part_file_mode) as new_part_file:
old_part_file = (open(old_part_file_name, 'r+b')
if old_part_file_name else None)
try:
self._ApplyOperations(operations, base_name, old_part_file,
new_part_file, new_part_info.size)
finally:
if old_part_file:
old_part_file.close()
# Truncate the result, if so instructed.
if self.truncate_to_expected_size:
new_part_file.seek(0, 2)
if new_part_file.tell() > new_part_info.size:
new_part_file.seek(new_part_info.size)
new_part_file.truncate()
# Verify the resulting partition.
with open(new_part_file_name, 'rb') as new_part_file:
_VerifySha256(new_part_file,
new_part_info.hash,
'new ' + part_name,
length=new_part_info.size)
def _ApplyDiffOperation(self, op, op_name, patch_data, old_part_file,
new_part_file):
"""Applies a SOURCE_BSDIFF or IMGDIFF operation.
Args:
op: the operation object
op_name: name string for error reporting
patch_data: the binary patch content
old_part_file: the source partition file object
new_part_file: the target partition file object
Raises:
PayloadError if something goes wrong.
"""
if not old_part_file:
raise PayloadError(
'%s: no source partition file provided for operation type (%d)'
% (op_name, op.type))
block_size = self.block_size
# Dump patch data to file.
with tempfile.NamedTemporaryFile(delete=False) as patch_file:
patch_file_name = patch_file.name
patch_file.write(patch_data)
if (hasattr(new_part_file, 'fileno')
and ((not old_part_file) or hasattr(old_part_file, 'fileno'))
and op.type != common.OpType.IMGDIFF):
# Construct input and output extents argument for bspatch.
in_extents_arg, _, _ = _ExtentsToBspatchArg(
op.src_extents,
block_size,
'%s.src_extents' % op_name,
data_length=op.src_length)
out_extents_arg, pad_off, pad_len = _ExtentsToBspatchArg(
op.dst_extents,
block_size,
'%s.dst_extents' % op_name,
data_length=op.dst_length)
new_file_name = '/dev/fd/%d' % new_part_file.fileno()
# Diff from source partition.
old_file_name = '/dev/fd/%d' % old_part_file.fileno()
# Invoke bspatch on partition file with extents args.
bspatch_cmd = [
self.bspatch_path, old_file_name, new_file_name,
patch_file_name, in_extents_arg, out_extents_arg
]
subprocess.check_call(bspatch_cmd)
# Pad with zeros past the total output length.
if pad_len:
new_part_file.seek(pad_off)
new_part_file.write('\0' * pad_len)
else:
# Gather input raw data and write to a temp file.
input_part_file = old_part_file if old_part_file else new_part_file
in_data = _ReadExtents(input_part_file,
op.src_extents,
block_size,
max_length=op.src_length)
with tempfile.NamedTemporaryFile(delete=False) as in_file:
in_file_name = in_file.name
in_file.write(in_data)
# Allocate temporary output file.
with tempfile.NamedTemporaryFile(delete=False) as out_file:
out_file_name = out_file.name
# Invoke bspatch.
patch_cmd = [
self.bspatch_path, in_file_name, out_file_name, patch_file_name
]
if op.type == common.OpType.IMGDIFF:
patch_cmd[0] = self.imgpatch_path
subprocess.check_call(patch_cmd)
# Read output.
with open(out_file_name, 'rb') as out_file:
out_data = out_file.read()
if len(out_data) != op.dst_length:
raise PayloadError(
'%s: actual patched data length (%d) not as expected (%d)'
% (op_name, len(out_data), op.dst_length))
# Write output back to partition, with padding.
unaligned_out_len = len(out_data) % block_size
if unaligned_out_len:
out_data += '\0' * (block_size - unaligned_out_len)
_WriteExtents(new_part_file, out_data, op.dst_extents, block_size,
'%s.dst_extents' % op_name)
# Delete input/output files.
os.remove(in_file_name)
os.remove(out_file_name)
# Delete patch file.
os.remove(patch_file_name)
def _ApplyReplaceOperation(self, op, op_name, out_data, part_file,
part_size):
"""Applies a REPLACE{,_BZ} operation.
Args:
op: the operation object
op_name: name string for error reporting
out_data: the data to be written
part_file: the partition file object
part_size: the size of the partition
Raises:
PayloadError if something goes wrong.
"""
block_size = self.block_size
data_length = len(out_data)
# Decompress data if needed.
if op.type == common.OpType.REPLACE_BZ:
out_data = bz2.decompress(out_data)
data_length = len(out_data)
# Write data to blocks specified in dst extents.
data_start = 0
for ex, ex_name in common.ExtentIter(op.dst_extents,
'%s.dst_extents' % op_name):
start_block = ex.start_block
num_blocks = ex.num_blocks
count = num_blocks * block_size
# Make sure it's not a fake (signature) operation.
if start_block != common.PSEUDO_EXTENT_MARKER:
data_end = data_start + count
# Make sure we're not running past partition boundary.
if (start_block + num_blocks) * block_size > part_size:
raise PayloadError(
'%s: extent (%s) exceeds partition size (%d)' %
(ex_name, common.FormatExtent(ex,
block_size), part_size))
# Make sure that we have enough data to write.
if data_end >= data_length + block_size:
raise PayloadError(
'%s: more dst blocks than data (even with padding)')
# Pad with zeros if necessary.
if data_end > data_length:
padding = data_end - data_length
out_data += '\0' * padding
self.payload.payload_file.seek(start_block * block_size)
part_file.seek(start_block * block_size)
part_file.write(out_data[data_start:data_end])
data_start += count
# Make sure we wrote all data.
if data_start < data_length:
raise PayloadError(
'%s: wrote fewer bytes (%d) than expected (%d)' %
(op_name, data_start, data_length))