def run(self, connection, max_end_time):
connection = super().run(connection, max_end_time)
# copy recovery image to a temporary directory and unpack
recovery_image = self.get_namespace_data(action="download-action",
label=self.param_key,
key="file")
recovery_image_dir = self.mkdtemp()
shutil.copy(recovery_image, recovery_image_dir)
tmp_recovery_image = os.path.join(recovery_image_dir,
os.path.basename(recovery_image))
if os.path.isfile(tmp_recovery_image):
if self.compression == "zip":
decompress_file(tmp_recovery_image, self.compression)
elif self.compression == "gz":
untar_file(tmp_recovery_image, recovery_image_dir)
else:
raise InfrastructureError(
"Unsupported compression for VExpress recovery: %s" %
self.compression)
os.remove(tmp_recovery_image)
self.set_namespace_data(
action="extract-vexpress-recovery-image",
label="file",
key=self.file_key,
value=recovery_image_dir,
)
self.logger.debug("Extracted %s to %s", self.file_key,
recovery_image_dir)
else:
raise InfrastructureError("Unable to decompress recovery image")
return connection
def test_multiple_decompressions(self):
"""
Previously had an issue with decompress_command_map being modified.
This should be a constant. If this is modified during calling decompress_file
then a regression has occurred.
:return:
"""
# Take a complete copy of decompress_command_map before it has been modified
copy_of_command_map = copy.deepcopy(decompress_command_map)
# Call decompress_file, we only need it to create the command required,
# it doesn't need to complete successfully.
with self.assertRaises(InfrastructureError):
decompress_file("/tmp/test.xz", "zip") # nosec - unit test only.
self.assertEqual(copy_of_command_map, decompress_command_map)
def copy_in_overlay(image, root_partition, overlay):
"""
Mounts test image partition as specified by the test
writer and extracts overlay at the root, if root_partition
is None the image is handled as a filesystem instead of
partitioned image.
"""
guest = guestfs.GuestFS(python_return_dict=True)
guest.add_drive(image)
guest.launch()
if root_partition:
partitions = guest.list_partitions()
if not partitions:
raise InfrastructureError("Unable to prepare guestfs")
guest_partition = partitions[root_partition]
guest.mount(guest_partition, '/')
else:
devices = guest.list_devices()
if not devices:
raise InfrastructureError("Unable to prepare guestfs")
guest.mount(devices[0], '/')
# FIXME: max message length issues when using tar_in
# on tar.gz. Works fine with tar so decompressing
# overlay first.
if os.path.exists(overlay[:-3]):
os.unlink(overlay[:-3])
decompressed_overlay = decompress_file(overlay, 'gz')
guest.tar_in(decompressed_overlay, '/')
if root_partition:
guest.umount(guest_partition)
else:
guest.umount(devices[0])
def run(self, connection, max_end_time, args=None):
if not self.parameters.get('ramdisk', None): # idempotency
return connection
ramdisk = self.get_namespace_data(action='download-action',
label='ramdisk',
key='file')
if self.skip:
self.logger.info("Not extracting ramdisk.")
suffix = self.get_namespace_data(action='tftp-deploy',
label='tftp',
key='suffix')
filename = os.path.join(suffix, "ramdisk",
os.path.basename(ramdisk))
# declare the original ramdisk as the name to be used later.
self.set_namespace_data(action='compress-ramdisk',
label='file',
key='ramdisk',
value=filename)
return
connection = super(ExtractRamdisk, self).run(connection, max_end_time,
args)
ramdisk_dir = self.mkdtemp()
extracted_ramdisk = os.path.join(ramdisk_dir, 'ramdisk')
os.mkdir(extracted_ramdisk, 0o755)
compression = self.parameters['ramdisk'].get('compression', None)
suffix = ""
if compression:
suffix = ".%s" % compression
ramdisk_compressed_data = os.path.join(ramdisk_dir,
RAMDISK_FNAME + suffix)
if self.parameters['ramdisk'].get('header', None) == 'u-boot':
cmd = ('dd if=%s of=%s ibs=%s skip=1' %
(ramdisk, ramdisk_compressed_data,
UBOOT_DEFAULT_HEADER_LENGTH)).split(' ')
try:
self.run_command(cmd)
except Exception:
raise LAVABug('Unable to remove uboot header: %s' % ramdisk)
else:
# give the file a predictable name
shutil.move(ramdisk, ramdisk_compressed_data)
ramdisk_data = decompress_file(ramdisk_compressed_data, compression)
with chdir(extracted_ramdisk):
cmd = ('cpio -iud -F %s' % ramdisk_data).split(' ')
if not self.run_command(cmd):
raise JobError(
'Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?'
% ramdisk_data)
# tell other actions where the unpacked ramdisk can be found
self.set_namespace_data(action=self.name,
label='extracted_ramdisk',
key='directory',
value=extracted_ramdisk)
self.set_namespace_data(action=self.name,
label='ramdisk_file',
key='file',
value=ramdisk_data)
return connection
def copy_overlay_to_sparse_fs(image, overlay):
"""copy_overlay_to_sparse_fs
Only copies the overlay to an image
which has already been converted from sparse.
"""
logger = logging.getLogger("dispatcher")
guest = guestfs.GuestFS(python_return_dict=True)
guest.add_drive(image)
_launch_guestfs(guest)
devices = guest.list_devices()
if not devices:
raise InfrastructureError("Unable to prepare guestfs")
guest.mount(devices[0], "/")
# FIXME: max message length issues when using tar_in
# on tar.gz. Works fine with tar so decompressing
# overlay first.
if os.path.exists(overlay[:-3]):
os.unlink(overlay[:-3])
decompressed_overlay = decompress_file(overlay, "gz")
guest.tar_in(decompressed_overlay, "/")
# Check if we have space left on the mounted image.
output = guest.df()
logger.debug(output)
_, _, _, available, percent, _ = output.split("\n")[1].split()
guest.umount(devices[0])
if int(available) == 0 or percent == "100%":
raise JobError("No space in image after applying overlay: %s" % image)
def run(self, connection, max_end_time):
if not self.parameters.get("ramdisk"): # idempotency
return connection
ramdisk = self.get_namespace_data(
action="download-action", label="ramdisk", key="file"
)
if self.skip:
self.logger.info("Not extracting ramdisk.")
suffix = self.get_namespace_data(
action="tftp-deploy", label="tftp", key="suffix"
)
filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk))
# declare the original ramdisk as the name to be used later.
self.set_namespace_data(
action="compress-ramdisk", label="file", key="ramdisk", value=filename
)
return connection
connection = super().run(connection, max_end_time)
ramdisk_dir = self.mkdtemp()
extracted_ramdisk = os.path.join(ramdisk_dir, "ramdisk")
os.mkdir(extracted_ramdisk, 0o755)
compression = self.parameters["ramdisk"].get("compression")
suffix = ""
if compression:
suffix = ".%s" % compression
ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix)
if self.parameters["ramdisk"].get("header") == "u-boot":
cmd = (
"dd if=%s of=%s ibs=%s skip=1"
% (ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH)
).split(" ")
try:
self.run_command(cmd)
except Exception:
raise LAVABug("Unable to remove uboot header: %s" % ramdisk)
else:
# give the file a predictable name
shutil.move(ramdisk, ramdisk_compressed_data)
ramdisk_data = decompress_file(ramdisk_compressed_data, compression)
with chdir(extracted_ramdisk):
cmd = ("cpio -iud -F %s" % ramdisk_data).split(" ")
if not self.run_command(cmd):
raise JobError(
"Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?"
% ramdisk_data
)
# tell other actions where the unpacked ramdisk can be found
self.set_namespace_data(
action=self.name,
label="extracted_ramdisk",
key="directory",
value=extracted_ramdisk,
)
self.set_namespace_data(
action=self.name, label="ramdisk_file", key="file", value=ramdisk_data
)
return connection
def _update(self, f_uncompress, f_compress):
image = self.get_namespace_data(action="download-action",
label=self.key,
key="file")
compression = self.get_namespace_data(action="download-action",
label=self.key,
key="compression")
decompressed = self.get_namespace_data(action="download-action",
label=self.key,
key="decompressed")
self.logger.info("Modifying %r", image)
tempdir = self.mkdtemp()
# Some images are kept compressed. We should decompress first
if compression and not decompressed:
self.logger.debug("* decompressing (%s)", compression)
image = decompress_file(image, compression)
# extract the archive
self.logger.debug("* extracting %r", image)
f_uncompress(image, tempdir)
os.unlink(image)
# Add overlays
self.logger.debug("Overlays:")
for overlay in self.params["overlays"]:
label = "%s.%s" % (self.key, overlay)
overlay_image = None
path = None
if overlay == "lava":
overlay_image = self.get_namespace_data(
action="compress-overlay", label="output", key="file")
path = "/"
else:
overlay_image = self.get_namespace_data(
action="download-action", label=label, key="file")
path = self.params["overlays"][overlay]["path"]
# Take off initial "/" from path, extract relative to this directory
extract_path = os.path.join(tempdir, path[1:])
if overlay == "lava" or self.params["overlays"][overlay][
"format"] == "tar":
self.logger.debug("- %s: untar %r to %r", label, overlay_image,
extract_path)
# In the "validate" function, we check that path startswith '/'
# and does not contains '..'
untar_file(overlay_image, extract_path)
else:
self.logger.debug("- %s: cp %r to %r", label, overlay_image,
extract_path)
shutil.copy(overlay_image, extract_path)
# Recreating the archive
self.logger.debug("* archiving %r", image)
f_compress(tempdir, image)
if compression and not decompressed:
self.logger.debug("* compressing (%s)", compression)
image = compress_file(image, compression)
def run(self, connection, max_end_time, args=None):
connection = super(ExtractVExpressRecoveryImage, self).run(connection, max_end_time, args)
# copy recovery image to a temporary directory and unpack
recovery_image = self.get_namespace_data(action='download-action', label=self.param_key, key='file')
recovery_image_dir = self.mkdtemp()
shutil.copy(recovery_image, recovery_image_dir)
tmp_recovery_image = os.path.join(recovery_image_dir, os.path.basename(recovery_image))
if os.path.isfile(tmp_recovery_image):
if self.compression == "zip":
decompress_file(tmp_recovery_image, self.compression)
elif self.compression == "gz":
untar_file(tmp_recovery_image, recovery_image_dir)
else:
raise InfrastructureError("Unsupported compression for VExpress recovery: %s" % self.compression)
os.remove(tmp_recovery_image)
self.set_namespace_data(action='extract-vexpress-recovery-image', label='file', key=self.file_key, value=recovery_image_dir)
self.logger.debug("Extracted %s to %s", self.file_key, recovery_image_dir)
else:
raise InfrastructureError("Unable to decompress recovery image")
return connection
def run(self, connection, max_end_time, args=None):
if not self.parameters.get('ramdisk', None): # idempotency
return connection
ramdisk = self.get_namespace_data(action='download-action', label='ramdisk', key='file')
if self.skip:
self.logger.info("Not extracting ramdisk.")
suffix = self.get_namespace_data(action='tftp-deploy', label='tftp', key='suffix')
filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk))
# declare the original ramdisk as the name to be used later.
self.set_namespace_data(action='compress-ramdisk', label='file', key='ramdisk', value=filename)
return
connection = super(ExtractRamdisk, self).run(connection, max_end_time, args)
ramdisk_dir = self.mkdtemp()
extracted_ramdisk = os.path.join(ramdisk_dir, 'ramdisk')
os.mkdir(extracted_ramdisk, 0o755)
compression = self.parameters['ramdisk'].get('compression', None)
suffix = ""
if compression:
suffix = ".%s" % compression
ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix)
if self.parameters['ramdisk'].get('header', None) == 'u-boot':
cmd = ('dd if=%s of=%s ibs=%s skip=1' % (
ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH)).split(' ')
try:
self.run_command(cmd)
except Exception:
raise LAVABug('Unable to remove uboot header: %s' % ramdisk)
else:
# give the file a predictable name
shutil.move(ramdisk, ramdisk_compressed_data)
ramdisk_data = decompress_file(ramdisk_compressed_data, compression)
with chdir(extracted_ramdisk):
cmd = ('cpio -iud -F %s' % ramdisk_data).split(' ')
if not self.run_command(cmd):
raise JobError('Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?' % ramdisk_data)
# tell other actions where the unpacked ramdisk can be found
self.set_namespace_data(action=self.name, label='extracted_ramdisk', key='directory', value=extracted_ramdisk)
self.set_namespace_data(action=self.name, label='ramdisk_file', key='file', value=ramdisk_data)
return connection
def copy_overlay_to_sparse_fs(image, overlay):
"""copy_overlay_to_sparse_fs
"""
ext4_img = image + '.ext4'
logger = logging.getLogger('dispatcher')
guest = guestfs.GuestFS(python_return_dict=True)
# Check if the given image is an Android sparse image
if not is_sparse_image(image):
raise JobError("Image is not an Android sparse image: %s" % image)
subprocess.check_output(
['/usr/bin/simg2img', image, ext4_img], # nosec - internal use.
stderr=subprocess.STDOUT)
guest.add_drive(ext4_img)
_launch_guestfs(guest)
devices = guest.list_devices()
if not devices:
raise InfrastructureError("Unable to prepare guestfs")
guest.mount(devices[0], '/')
# FIXME: max message length issues when using tar_in
# on tar.gz. Works fine with tar so decompressing
# overlay first.
if os.path.exists(overlay[:-3]):
os.unlink(overlay[:-3])
decompressed_overlay = decompress_file(overlay, 'gz')
guest.tar_in(decompressed_overlay, '/')
# Check if we have space left on the mounted image.
output = guest.df()
logger.debug(output)
_, _, _, available, percent, _ = output.split("\n")[1].split()
guest.umount(devices[0])
if int(available) is 0 or percent == '100%':
raise JobError("No space in image after applying overlay: %s" % image)
subprocess.check_output(
['/usr/bin/img2simg', ext4_img, image], # nosec - internal use.
stderr=subprocess.STDOUT)
os.remove(ext4_img)
