def run(self, connection, max_end_time, args=None): output = os.path.join(self.mkdtemp(), "overlay-%s.tar.gz" % self.level) location = self.get_namespace_data(action='test', label='shared', key='location') lava_test_results_dir = self.get_namespace_data(action='test', label='results', key='lava_test_results_dir') self.set_namespace_data(action='test', label='shared', key='output', value=output) if not location: raise LAVABug("Missing lava overlay location") if not os.path.exists(location): raise LAVABug("Unable to find overlay location") if not self.valid: self.logger.error(self.errors) return connection connection = super(CompressOverlay, self).run(connection, max_end_time, args) with chdir(location): try: with tarfile.open(output, "w:gz") as tar: tar.add(".%s" % lava_test_results_dir) # ssh authorization support if os.path.exists('./root/'): tar.add(".%s" % '/root/') except tarfile.TarError as exc: raise InfrastructureError("Unable to create lava overlay tarball: %s" % exc) self.set_namespace_data(action=self.name, label='output', key='file', value=output) return connection
def run(self, connection, max_end_time): output = os.path.join(self.mkdtemp(), "overlay-%s.tar.gz" % self.level) location = self.get_namespace_data(action='test', label='shared', key='location') lava_test_results_dir = self.get_namespace_data( action='test', label='results', key='lava_test_results_dir') self.set_namespace_data(action='test', label='shared', key='output', value=output) if not location: raise LAVABug("Missing lava overlay location") if not os.path.exists(location): raise LAVABug("Unable to find overlay location") if not self.valid: self.logger.error(self.errors) return connection connection = super().run(connection, max_end_time) with chdir(location): try: with tarfile.open(output, "w:gz") as tar: tar.add(".%s" % lava_test_results_dir) # ssh authorization support if os.path.exists('./root/'): tar.add(".%s" % '/root/') except tarfile.TarError as exc: raise InfrastructureError( "Unable to create lava overlay tarball: %s" % exc) self.set_namespace_data(action=self.name, label='output', key='file', value=output) return connection
def run(self, connection, max_end_time, args=None): if not self.parameters.get('ramdisk', None): # idempotency return connection ramdisk = self.get_namespace_data(action='download-action', label='ramdisk', key='file') if self.skip: self.logger.info("Not extracting ramdisk.") suffix = self.get_namespace_data(action='tftp-deploy', label='tftp', key='suffix') filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk)) # declare the original ramdisk as the name to be used later. self.set_namespace_data(action='compress-ramdisk', label='file', key='ramdisk', value=filename) return connection = super(ExtractRamdisk, self).run(connection, max_end_time, args) ramdisk_dir = self.mkdtemp() extracted_ramdisk = os.path.join(ramdisk_dir, 'ramdisk') os.mkdir(extracted_ramdisk, 0o755) compression = self.parameters['ramdisk'].get('compression', None) suffix = "" if compression: suffix = ".%s" % compression ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix) if self.parameters['ramdisk'].get('header', None) == 'u-boot': cmd = ('dd if=%s of=%s ibs=%s skip=1' % (ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH)).split(' ') try: self.run_command(cmd) except Exception: raise LAVABug('Unable to remove uboot header: %s' % ramdisk) else: # give the file a predictable name shutil.move(ramdisk, ramdisk_compressed_data) ramdisk_data = decompress_file(ramdisk_compressed_data, compression) with chdir(extracted_ramdisk): cmd = ('cpio -iud -F %s' % ramdisk_data).split(' ') if not self.run_command(cmd): raise JobError( 'Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?' % ramdisk_data) # tell other actions where the unpacked ramdisk can be found self.set_namespace_data(action=self.name, label='extracted_ramdisk', key='directory', value=extracted_ramdisk) self.set_namespace_data(action=self.name, label='ramdisk_file', key='file', value=ramdisk_data) return connection
def run(self, connection, max_end_time): if not self.parameters.get("ramdisk"): # idempotency return connection ramdisk = self.get_namespace_data( action="download-action", label="ramdisk", key="file" ) if self.skip: self.logger.info("Not extracting ramdisk.") suffix = self.get_namespace_data( action="tftp-deploy", label="tftp", key="suffix" ) filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk)) # declare the original ramdisk as the name to be used later. self.set_namespace_data( action="compress-ramdisk", label="file", key="ramdisk", value=filename ) return connection connection = super().run(connection, max_end_time) ramdisk_dir = self.mkdtemp() extracted_ramdisk = os.path.join(ramdisk_dir, "ramdisk") os.mkdir(extracted_ramdisk, 0o755) compression = self.parameters["ramdisk"].get("compression") suffix = "" if compression: suffix = ".%s" % compression ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix) if self.parameters["ramdisk"].get("header") == "u-boot": cmd = ( "dd if=%s of=%s ibs=%s skip=1" % (ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH) ).split(" ") try: self.run_command(cmd) except Exception: raise LAVABug("Unable to remove uboot header: %s" % ramdisk) else: # give the file a predictable name shutil.move(ramdisk, ramdisk_compressed_data) ramdisk_data = decompress_file(ramdisk_compressed_data, compression) with chdir(extracted_ramdisk): cmd = ("cpio -iud -F %s" % ramdisk_data).split(" ") if not self.run_command(cmd): raise JobError( "Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?" % ramdisk_data ) # tell other actions where the unpacked ramdisk can be found self.set_namespace_data( action=self.name, label="extracted_ramdisk", key="directory", value=extracted_ramdisk, ) self.set_namespace_data( action=self.name, label="ramdisk_file", key="file", value=ramdisk_data ) return connection
def cpio(directory, filename): which("cpio") which("find") with chdir(directory): try: find = subprocess.check_output(["find", "."], stderr=subprocess.STDOUT) # nosec return subprocess.check_output( # nosec ["cpio", "--create", "--format", "newc", "--file", filename], input=find, stderr=subprocess.STDOUT, ).decode("utf-8", errors="replace") except Exception as exc: raise InfrastructureError("Unable to create cpio archive %r: %s" % (filename, exc))
def compress_file(infile, compression): if not compression: return infile if compression not in compress_command_map.keys(): raise JobError("Cannot find shell command to compress: %s" % compression) # Check that the command does exists which(compress_command_map[compression][0]) with chdir(os.path.dirname(infile)): # local copy for idempotency cmd = compress_command_map[compression][:] cmd.append(infile) try: subprocess.check_output(cmd) # nosec - internal use. return "%s.%s" % (infile, compression) except (OSError, subprocess.CalledProcessError) as exc: raise InfrastructureError("unable to compress file %s: %s" % (infile, exc))
def uncpio(filename, directory): which("cpio") with chdir(directory): try: subprocess.check_output( # nosec [ "cpio", "--extract", "--make-directories", "--unconditional", "--file", filename, ], stderr=subprocess.STDOUT, ) except subprocess.SubprocessError as exc: raise InfrastructureError("Unable to extract cpio archive %r: %s" % (filename, exc))
def run(self, connection, max_end_time): output = os.path.join(self.mkdtemp(), "overlay-%s.tar.gz" % self.level) location = self.get_namespace_data(action="test", label="shared", key="location") lava_test_results_dir = self.get_namespace_data( action="test", label="results", key="lava_test_results_dir") self.set_namespace_data(action="test", label="shared", key="output", value=output) if not location: raise LAVABug("Missing lava overlay location") if not os.path.exists(location): raise LAVABug("Unable to find overlay location") if not self.valid: self.logger.error(self.errors) return connection connection = super().run(connection, max_end_time) # This will split the path into two parts. Second part is only the # bottom level directory. The first part is the rest. # Example: /run/mount/lava/ will be split to /run/mount/ and lava/ # Used to create a tarball with only the bottom level dir included and # not with the whole lava_test_results_dir property. results_dir_list = os.path.split( os.path.normpath(lava_test_results_dir)) with chdir( os.path.join(location, os.path.relpath(results_dir_list[0], os.sep))): try: with tarfile.open(output, "w:gz") as tar: tar.add("%s" % results_dir_list[1]) # ssh authorization support if os.path.exists("./root/"): tar.add(".%s" % "/root/") except tarfile.TarError as exc: raise InfrastructureError( "Unable to create lava overlay tarball: %s" % exc) self.set_namespace_data(action=self.name, label="output", key="file", value=output) return connection
def compress_file(infile, compression): if not compression: return infile if compression not in compress_command_map.keys(): raise JobError("Cannot find shell command to compress: %s" % compression) # Check that the command does exists which(compress_command_map[compression].split(" ")[0]) with chdir(os.path.dirname(infile)): cmd = "%s %s" % (compress_command_map[compression], infile) try: # safe to use shell=True here, no external arguments subprocess.check_output(cmd, shell=True) return "%s.%s" % (infile, compression) except (OSError, subprocess.CalledProcessError) as exc: raise InfrastructureError('unable to compress file %s: %s' % (infile, exc))
def run(self, connection, max_end_time, args=None): if not self.parameters.get('ramdisk', None): # idempotency return connection ramdisk = self.get_namespace_data(action='download-action', label='ramdisk', key='file') if self.skip: self.logger.info("Not extracting ramdisk.") suffix = self.get_namespace_data(action='tftp-deploy', label='tftp', key='suffix') filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk)) # declare the original ramdisk as the name to be used later. self.set_namespace_data(action='compress-ramdisk', label='file', key='ramdisk', value=filename) return connection = super(ExtractRamdisk, self).run(connection, max_end_time, args) ramdisk_dir = self.mkdtemp() extracted_ramdisk = os.path.join(ramdisk_dir, 'ramdisk') os.mkdir(extracted_ramdisk, 0o755) compression = self.parameters['ramdisk'].get('compression', None) suffix = "" if compression: suffix = ".%s" % compression ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix) if self.parameters['ramdisk'].get('header', None) == 'u-boot': cmd = ('dd if=%s of=%s ibs=%s skip=1' % ( ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH)).split(' ') try: self.run_command(cmd) except Exception: raise LAVABug('Unable to remove uboot header: %s' % ramdisk) else: # give the file a predictable name shutil.move(ramdisk, ramdisk_compressed_data) ramdisk_data = decompress_file(ramdisk_compressed_data, compression) with chdir(extracted_ramdisk): cmd = ('cpio -iud -F %s' % ramdisk_data).split(' ') if not self.run_command(cmd): raise JobError('Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?' % ramdisk_data) # tell other actions where the unpacked ramdisk can be found self.set_namespace_data(action=self.name, label='extracted_ramdisk', key='directory', value=extracted_ramdisk) self.set_namespace_data(action=self.name, label='ramdisk_file', key='file', value=ramdisk_data) return connection
def run(self, connection, max_end_time, args=None): # pylint: disable=too-many-locals if not self.parameters.get('ramdisk', None): # idempotency return connection if self.skip: return connection connection = super(CompressRamdisk, self).run(connection, max_end_time, args) ramdisk_dir = self.get_namespace_data(action='extract-overlay-ramdisk', label='extracted_ramdisk', key='directory') ramdisk_data = self.get_namespace_data( action='extract-overlay-ramdisk', label='ramdisk_file', key='file') if not ramdisk_dir: raise LAVABug("Unable to find unpacked ramdisk") if not ramdisk_data: raise LAVABug("Unable to find ramdisk directory") if self.parameters.get('preseed', None): if self.parameters["deployment_data"].get("preseed_to_ramdisk", None): # download action must have completed to get this far # some installers (centos) cannot fetch the preseed file via tftp. # Instead, put the preseed file into the ramdisk using a given name # from deployment_data which we can use in the boot commands. filename = self.parameters["deployment_data"][ "preseed_to_ramdisk"] self.logger.info("Copying preseed file into ramdisk: %s", filename) shutil.copy( self.get_namespace_data(action='download-action', label='preseed', key='file'), os.path.join(ramdisk_dir, filename)) self.set_namespace_data(action=self.name, label='file', key='preseed_local', value=filename) with chdir(ramdisk_dir): self.logger.info("Building ramdisk %s containing %s", ramdisk_data, ramdisk_dir) cmd = "find . | cpio --create --format='newc' > %s" % ramdisk_data try: # safe to use shell=True here, no external arguments log = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) except OSError as exc: raise InfrastructureError( 'Unable to create cpio filesystem: %s' % exc) # lazy-logging would mean that the quoting of cmd causes invalid YAML self.logger.debug("%s\n%s" % (cmd, log)) # pylint: disable=logging-not-lazy # we need to compress the ramdisk with the same method is was submitted with compression = self.parameters['ramdisk'].get('compression', None) final_file = compress_file(ramdisk_data, compression) tftp_dir = os.path.dirname( self.get_namespace_data(action='download-action', label='ramdisk', key='file')) if self.add_header == 'u-boot': ramdisk_uboot = final_file + ".uboot" self.logger.debug("Adding RAMdisk u-boot header.") cmd = ("mkimage -A %s -T ramdisk -C none -d %s %s" % (self.mkimage_arch, final_file, ramdisk_uboot)).split(' ') if not self.run_command(cmd): raise InfrastructureError( "Unable to add uboot header to ramdisk") final_file = ramdisk_uboot full_path = os.path.join(tftp_dir, os.path.basename(final_file)) shutil.move(final_file, full_path) self.logger.debug("rename %s to %s", final_file, full_path) self.set_namespace_data(action=self.name, label='file', key='full-path', value=full_path) if self.parameters['to'] == 'tftp': suffix = self.get_namespace_data(action='tftp-deploy', label='tftp', key='suffix') if not suffix: suffix = '' self.set_namespace_data(action=self.name, label='file', key='ramdisk', value=os.path.join( suffix, "ramdisk", os.path.basename(final_file))) else: self.set_namespace_data(action=self.name, label='file', key='ramdisk', value=final_file) return connection
def run(self, connection, max_end_time): if not self.parameters.get("ramdisk"): # idempotency return connection if self.skip: return connection connection = super().run(connection, max_end_time) ramdisk_dir = self.get_namespace_data(action="extract-overlay-ramdisk", label="extracted_ramdisk", key="directory") ramdisk_data = self.get_namespace_data( action="extract-overlay-ramdisk", label="ramdisk_file", key="file") if not ramdisk_dir: raise LAVABug("Unable to find unpacked ramdisk") if not ramdisk_data: raise LAVABug("Unable to find ramdisk directory") if self.parameters.get("preseed"): if self.parameters["deployment_data"].get("preseed_to_ramdisk"): # download action must have completed to get this far # some installers (centos) cannot fetch the preseed file via tftp. # Instead, put the preseed file into the ramdisk using a given name # from deployment_data which we can use in the boot commands. filename = self.parameters["deployment_data"][ "preseed_to_ramdisk"] self.logger.info("Copying preseed file into ramdisk: %s", filename) shutil.copy( self.get_namespace_data(action="download-action", label="preseed", key="file"), os.path.join(ramdisk_dir, filename), ) self.set_namespace_data(action=self.name, label="file", key="preseed_local", value=filename) with chdir(ramdisk_dir): self.logger.info("Building ramdisk %s containing %s", ramdisk_data, ramdisk_dir) cmd = "find . | cpio --create --format='newc' > %s" % ramdisk_data try: log = subprocess.check_output( # nosec - safe to use shell=True here, no external arguments cmd, shell=True, stderr=subprocess.STDOUT) log = log.decode("utf-8", errors="replace") except OSError as exc: raise InfrastructureError( "Unable to create cpio filesystem: %s" % exc) # lazy-logging would mean that the quoting of cmd causes invalid YAML self.logger.debug("%s\n%s" % (cmd, log)) # we need to compress the ramdisk with the same method is was submitted with compression = self.parameters["ramdisk"].get("compression") final_file = compress_file(ramdisk_data, compression) tftp_dir = os.path.dirname( self.get_namespace_data(action="download-action", label="ramdisk", key="file")) if self.add_header == "u-boot": ramdisk_uboot = final_file + ".uboot" self.logger.debug("Adding RAMdisk u-boot header.") cmd = ("mkimage -A %s -T ramdisk -C none -d %s %s" % (self.mkimage_arch, final_file, ramdisk_uboot)).split(" ") if not self.run_command(cmd): raise InfrastructureError( "Unable to add uboot header to ramdisk") final_file = ramdisk_uboot full_path = os.path.join(tftp_dir, os.path.basename(final_file)) shutil.move(final_file, full_path) self.logger.debug("rename %s to %s", final_file, full_path) self.set_namespace_data(action=self.name, label="file", key="full-path", value=full_path) if self.parameters["to"] == "tftp": suffix = self.get_namespace_data(action="tftp-deploy", label="tftp", key="suffix") if not suffix: suffix = "" self.set_namespace_data( action=self.name, label="file", key="ramdisk", value=os.path.join(suffix, "ramdisk", os.path.basename(final_file)), ) else: self.set_namespace_data(action=self.name, label="file", key="ramdisk", value=final_file) return connection
def run(self, connection, max_end_time, args=None): # pylint: disable=too-many-locals if not self.parameters.get('ramdisk', None): # idempotency return connection if self.skip: return connection connection = super(CompressRamdisk, self).run(connection, max_end_time, args) ramdisk_dir = self.get_namespace_data( action='extract-overlay-ramdisk', label='extracted_ramdisk', key='directory') ramdisk_data = self.get_namespace_data( action='extract-overlay-ramdisk', label='ramdisk_file', key='file') if not ramdisk_dir: raise LAVABug("Unable to find unpacked ramdisk") if not ramdisk_data: raise LAVABug("Unable to find ramdisk directory") if self.parameters.get('preseed', None): if self.parameters["deployment_data"].get("preseed_to_ramdisk", None): # download action must have completed to get this far # some installers (centos) cannot fetch the preseed file via tftp. # Instead, put the preseed file into the ramdisk using a given name # from deployment_data which we can use in the boot commands. filename = self.parameters["deployment_data"]["preseed_to_ramdisk"] self.logger.info("Copying preseed file into ramdisk: %s", filename) shutil.copy(self.get_namespace_data( action='download-action', label='preseed', key='file'), os.path.join(ramdisk_dir, filename)) self.set_namespace_data(action=self.name, label='file', key='preseed_local', value=filename) with chdir(ramdisk_dir): self.logger.info("Building ramdisk %s containing %s", ramdisk_data, ramdisk_dir) cmd = "find . | cpio --create --format='newc' > %s" % ramdisk_data try: # safe to use shell=True here, no external arguments log = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) except OSError as exc: raise InfrastructureError('Unable to create cpio filesystem: %s' % exc) # lazy-logging would mean that the quoting of cmd causes invalid YAML self.logger.debug("%s\n%s" % (cmd, log)) # pylint: disable=logging-not-lazy # we need to compress the ramdisk with the same method is was submitted with compression = self.parameters['ramdisk'].get('compression', None) final_file = compress_file(ramdisk_data, compression) tftp_dir = os.path.dirname(self.get_namespace_data( action='download-action', label='ramdisk', key='file')) if self.add_header == 'u-boot': ramdisk_uboot = final_file + ".uboot" self.logger.debug("Adding RAMdisk u-boot header.") cmd = ("mkimage -A %s -T ramdisk -C none -d %s %s" % (self.mkimage_arch, final_file, ramdisk_uboot)).split(' ') if not self.run_command(cmd): raise InfrastructureError("Unable to add uboot header to ramdisk") final_file = ramdisk_uboot full_path = os.path.join(tftp_dir, os.path.basename(final_file)) shutil.move(final_file, full_path) self.logger.debug("rename %s to %s", final_file, full_path) self.set_namespace_data( action=self.name, label='file', key='full-path', value=full_path) if self.parameters['to'] == 'tftp': suffix = self.get_namespace_data(action='tftp-deploy', label='tftp', key='suffix') if not suffix: suffix = '' self.set_namespace_data( action=self.name, label='file', key='ramdisk', value=os.path.join(suffix, "ramdisk", os.path.basename(final_file))) else: self.set_namespace_data( action=self.name, label='file', key='ramdisk', value=final_file) return connection