def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) # copy recovery image to a temporary directory and unpack recovery_image = self.get_namespace_data(action="download-action", label=self.param_key, key="file") recovery_image_dir = self.mkdtemp() shutil.copy(recovery_image, recovery_image_dir) tmp_recovery_image = os.path.join(recovery_image_dir, os.path.basename(recovery_image)) if os.path.isfile(tmp_recovery_image): if self.compression == "zip": decompress_file(tmp_recovery_image, self.compression) elif self.compression == "gz": untar_file(tmp_recovery_image, recovery_image_dir) else: raise InfrastructureError( "Unsupported compression for VExpress recovery: %s" % self.compression) os.remove(tmp_recovery_image) self.set_namespace_data( action="extract-vexpress-recovery-image", label="file", key=self.file_key, value=recovery_image_dir, ) self.logger.debug("Extracted %s to %s", self.file_key, recovery_image_dir) else: raise InfrastructureError("Unable to decompress recovery image") return connection
def test_multiple_decompressions(self): """ Previously had an issue with decompress_command_map being modified. This should be a constant. If this is modified during calling decompress_file then a regression has occurred. :return: """ # Take a complete copy of decompress_command_map before it has been modified copy_of_command_map = copy.deepcopy(decompress_command_map) # Call decompress_file, we only need it to create the command required, # it doesn't need to complete successfully. with self.assertRaises(InfrastructureError): decompress_file("/tmp/test.xz", "zip") # nosec - unit test only. self.assertEqual(copy_of_command_map, decompress_command_map)
def copy_in_overlay(image, root_partition, overlay): """ Mounts test image partition as specified by the test writer and extracts overlay at the root, if root_partition is None the image is handled as a filesystem instead of partitioned image. """ guest = guestfs.GuestFS(python_return_dict=True) guest.add_drive(image) guest.launch() if root_partition: partitions = guest.list_partitions() if not partitions: raise InfrastructureError("Unable to prepare guestfs") guest_partition = partitions[root_partition] guest.mount(guest_partition, '/') else: devices = guest.list_devices() if not devices: raise InfrastructureError("Unable to prepare guestfs") guest.mount(devices[0], '/') # FIXME: max message length issues when using tar_in # on tar.gz. Works fine with tar so decompressing # overlay first. if os.path.exists(overlay[:-3]): os.unlink(overlay[:-3]) decompressed_overlay = decompress_file(overlay, 'gz') guest.tar_in(decompressed_overlay, '/') if root_partition: guest.umount(guest_partition) else: guest.umount(devices[0])
def run(self, connection, max_end_time, args=None): if not self.parameters.get('ramdisk', None): # idempotency return connection ramdisk = self.get_namespace_data(action='download-action', label='ramdisk', key='file') if self.skip: self.logger.info("Not extracting ramdisk.") suffix = self.get_namespace_data(action='tftp-deploy', label='tftp', key='suffix') filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk)) # declare the original ramdisk as the name to be used later. self.set_namespace_data(action='compress-ramdisk', label='file', key='ramdisk', value=filename) return connection = super(ExtractRamdisk, self).run(connection, max_end_time, args) ramdisk_dir = self.mkdtemp() extracted_ramdisk = os.path.join(ramdisk_dir, 'ramdisk') os.mkdir(extracted_ramdisk, 0o755) compression = self.parameters['ramdisk'].get('compression', None) suffix = "" if compression: suffix = ".%s" % compression ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix) if self.parameters['ramdisk'].get('header', None) == 'u-boot': cmd = ('dd if=%s of=%s ibs=%s skip=1' % (ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH)).split(' ') try: self.run_command(cmd) except Exception: raise LAVABug('Unable to remove uboot header: %s' % ramdisk) else: # give the file a predictable name shutil.move(ramdisk, ramdisk_compressed_data) ramdisk_data = decompress_file(ramdisk_compressed_data, compression) with chdir(extracted_ramdisk): cmd = ('cpio -iud -F %s' % ramdisk_data).split(' ') if not self.run_command(cmd): raise JobError( 'Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?' % ramdisk_data) # tell other actions where the unpacked ramdisk can be found self.set_namespace_data(action=self.name, label='extracted_ramdisk', key='directory', value=extracted_ramdisk) self.set_namespace_data(action=self.name, label='ramdisk_file', key='file', value=ramdisk_data) return connection
def copy_overlay_to_sparse_fs(image, overlay): """copy_overlay_to_sparse_fs Only copies the overlay to an image which has already been converted from sparse. """ logger = logging.getLogger("dispatcher") guest = guestfs.GuestFS(python_return_dict=True) guest.add_drive(image) _launch_guestfs(guest) devices = guest.list_devices() if not devices: raise InfrastructureError("Unable to prepare guestfs") guest.mount(devices[0], "/") # FIXME: max message length issues when using tar_in # on tar.gz. Works fine with tar so decompressing # overlay first. if os.path.exists(overlay[:-3]): os.unlink(overlay[:-3]) decompressed_overlay = decompress_file(overlay, "gz") guest.tar_in(decompressed_overlay, "/") # Check if we have space left on the mounted image. output = guest.df() logger.debug(output) _, _, _, available, percent, _ = output.split("\n")[1].split() guest.umount(devices[0]) if int(available) == 0 or percent == "100%": raise JobError("No space in image after applying overlay: %s" % image)
def run(self, connection, max_end_time): if not self.parameters.get("ramdisk"): # idempotency return connection ramdisk = self.get_namespace_data( action="download-action", label="ramdisk", key="file" ) if self.skip: self.logger.info("Not extracting ramdisk.") suffix = self.get_namespace_data( action="tftp-deploy", label="tftp", key="suffix" ) filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk)) # declare the original ramdisk as the name to be used later. self.set_namespace_data( action="compress-ramdisk", label="file", key="ramdisk", value=filename ) return connection connection = super().run(connection, max_end_time) ramdisk_dir = self.mkdtemp() extracted_ramdisk = os.path.join(ramdisk_dir, "ramdisk") os.mkdir(extracted_ramdisk, 0o755) compression = self.parameters["ramdisk"].get("compression") suffix = "" if compression: suffix = ".%s" % compression ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix) if self.parameters["ramdisk"].get("header") == "u-boot": cmd = ( "dd if=%s of=%s ibs=%s skip=1" % (ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH) ).split(" ") try: self.run_command(cmd) except Exception: raise LAVABug("Unable to remove uboot header: %s" % ramdisk) else: # give the file a predictable name shutil.move(ramdisk, ramdisk_compressed_data) ramdisk_data = decompress_file(ramdisk_compressed_data, compression) with chdir(extracted_ramdisk): cmd = ("cpio -iud -F %s" % ramdisk_data).split(" ") if not self.run_command(cmd): raise JobError( "Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?" % ramdisk_data ) # tell other actions where the unpacked ramdisk can be found self.set_namespace_data( action=self.name, label="extracted_ramdisk", key="directory", value=extracted_ramdisk, ) self.set_namespace_data( action=self.name, label="ramdisk_file", key="file", value=ramdisk_data ) return connection
def _update(self, f_uncompress, f_compress): image = self.get_namespace_data(action="download-action", label=self.key, key="file") compression = self.get_namespace_data(action="download-action", label=self.key, key="compression") decompressed = self.get_namespace_data(action="download-action", label=self.key, key="decompressed") self.logger.info("Modifying %r", image) tempdir = self.mkdtemp() # Some images are kept compressed. We should decompress first if compression and not decompressed: self.logger.debug("* decompressing (%s)", compression) image = decompress_file(image, compression) # extract the archive self.logger.debug("* extracting %r", image) f_uncompress(image, tempdir) os.unlink(image) # Add overlays self.logger.debug("Overlays:") for overlay in self.params["overlays"]: label = "%s.%s" % (self.key, overlay) overlay_image = None path = None if overlay == "lava": overlay_image = self.get_namespace_data( action="compress-overlay", label="output", key="file") path = "/" else: overlay_image = self.get_namespace_data( action="download-action", label=label, key="file") path = self.params["overlays"][overlay]["path"] # Take off initial "/" from path, extract relative to this directory extract_path = os.path.join(tempdir, path[1:]) if overlay == "lava" or self.params["overlays"][overlay][ "format"] == "tar": self.logger.debug("- %s: untar %r to %r", label, overlay_image, extract_path) # In the "validate" function, we check that path startswith '/' # and does not contains '..' untar_file(overlay_image, extract_path) else: self.logger.debug("- %s: cp %r to %r", label, overlay_image, extract_path) shutil.copy(overlay_image, extract_path) # Recreating the archive self.logger.debug("* archiving %r", image) f_compress(tempdir, image) if compression and not decompressed: self.logger.debug("* compressing (%s)", compression) image = compress_file(image, compression)
def run(self, connection, max_end_time, args=None): connection = super(ExtractVExpressRecoveryImage, self).run(connection, max_end_time, args) # copy recovery image to a temporary directory and unpack recovery_image = self.get_namespace_data(action='download-action', label=self.param_key, key='file') recovery_image_dir = self.mkdtemp() shutil.copy(recovery_image, recovery_image_dir) tmp_recovery_image = os.path.join(recovery_image_dir, os.path.basename(recovery_image)) if os.path.isfile(tmp_recovery_image): if self.compression == "zip": decompress_file(tmp_recovery_image, self.compression) elif self.compression == "gz": untar_file(tmp_recovery_image, recovery_image_dir) else: raise InfrastructureError("Unsupported compression for VExpress recovery: %s" % self.compression) os.remove(tmp_recovery_image) self.set_namespace_data(action='extract-vexpress-recovery-image', label='file', key=self.file_key, value=recovery_image_dir) self.logger.debug("Extracted %s to %s", self.file_key, recovery_image_dir) else: raise InfrastructureError("Unable to decompress recovery image") return connection
def run(self, connection, max_end_time, args=None): if not self.parameters.get('ramdisk', None): # idempotency return connection ramdisk = self.get_namespace_data(action='download-action', label='ramdisk', key='file') if self.skip: self.logger.info("Not extracting ramdisk.") suffix = self.get_namespace_data(action='tftp-deploy', label='tftp', key='suffix') filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk)) # declare the original ramdisk as the name to be used later. self.set_namespace_data(action='compress-ramdisk', label='file', key='ramdisk', value=filename) return connection = super(ExtractRamdisk, self).run(connection, max_end_time, args) ramdisk_dir = self.mkdtemp() extracted_ramdisk = os.path.join(ramdisk_dir, 'ramdisk') os.mkdir(extracted_ramdisk, 0o755) compression = self.parameters['ramdisk'].get('compression', None) suffix = "" if compression: suffix = ".%s" % compression ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix) if self.parameters['ramdisk'].get('header', None) == 'u-boot': cmd = ('dd if=%s of=%s ibs=%s skip=1' % ( ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH)).split(' ') try: self.run_command(cmd) except Exception: raise LAVABug('Unable to remove uboot header: %s' % ramdisk) else: # give the file a predictable name shutil.move(ramdisk, ramdisk_compressed_data) ramdisk_data = decompress_file(ramdisk_compressed_data, compression) with chdir(extracted_ramdisk): cmd = ('cpio -iud -F %s' % ramdisk_data).split(' ') if not self.run_command(cmd): raise JobError('Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?' % ramdisk_data) # tell other actions where the unpacked ramdisk can be found self.set_namespace_data(action=self.name, label='extracted_ramdisk', key='directory', value=extracted_ramdisk) self.set_namespace_data(action=self.name, label='ramdisk_file', key='file', value=ramdisk_data) return connection
def copy_overlay_to_sparse_fs(image, overlay): """copy_overlay_to_sparse_fs """ ext4_img = image + '.ext4' logger = logging.getLogger('dispatcher') guest = guestfs.GuestFS(python_return_dict=True) # Check if the given image is an Android sparse image if not is_sparse_image(image): raise JobError("Image is not an Android sparse image: %s" % image) subprocess.check_output( ['/usr/bin/simg2img', image, ext4_img], # nosec - internal use. stderr=subprocess.STDOUT) guest.add_drive(ext4_img) _launch_guestfs(guest) devices = guest.list_devices() if not devices: raise InfrastructureError("Unable to prepare guestfs") guest.mount(devices[0], '/') # FIXME: max message length issues when using tar_in # on tar.gz. Works fine with tar so decompressing # overlay first. if os.path.exists(overlay[:-3]): os.unlink(overlay[:-3]) decompressed_overlay = decompress_file(overlay, 'gz') guest.tar_in(decompressed_overlay, '/') # Check if we have space left on the mounted image. output = guest.df() logger.debug(output) _, _, _, available, percent, _ = output.split("\n")[1].split() guest.umount(devices[0]) if int(available) is 0 or percent == '100%': raise JobError("No space in image after applying overlay: %s" % image) subprocess.check_output( ['/usr/bin/img2simg', ext4_img, image], # nosec - internal use. stderr=subprocess.STDOUT) os.remove(ext4_img)