def run(self, connection, max_end_time, args=None):
output = os.path.join(self.mkdtemp(), "overlay-%s.tar.gz" % self.level)
location = self.get_namespace_data(action='test', label='shared', key='location')
lava_test_results_dir = self.get_namespace_data(action='test', label='results', key='lava_test_results_dir')
self.set_namespace_data(action='test', label='shared', key='output', value=output)
if not location:
raise LAVABug("Missing lava overlay location")
if not os.path.exists(location):
raise LAVABug("Unable to find overlay location")
if not self.valid:
self.logger.error(self.errors)
return connection
connection = super(CompressOverlay, self).run(connection, max_end_time, args)
with chdir(location):
try:
with tarfile.open(output, "w:gz") as tar:
tar.add(".%s" % lava_test_results_dir)
# ssh authorization support
if os.path.exists('./root/'):
tar.add(".%s" % '/root/')
except tarfile.TarError as exc:
raise InfrastructureError("Unable to create lava overlay tarball: %s" % exc)
self.set_namespace_data(action=self.name, label='output', key='file', value=output)
return connection
def run(self, connection, max_end_time):
output = os.path.join(self.mkdtemp(), "overlay-%s.tar.gz" % self.level)
location = self.get_namespace_data(action='test',
label='shared',
key='location')
lava_test_results_dir = self.get_namespace_data(
action='test', label='results', key='lava_test_results_dir')
self.set_namespace_data(action='test',
label='shared',
key='output',
value=output)
if not location:
raise LAVABug("Missing lava overlay location")
if not os.path.exists(location):
raise LAVABug("Unable to find overlay location")
if not self.valid:
self.logger.error(self.errors)
return connection
connection = super().run(connection, max_end_time)
with chdir(location):
try:
with tarfile.open(output, "w:gz") as tar:
tar.add(".%s" % lava_test_results_dir)
# ssh authorization support
if os.path.exists('./root/'):
tar.add(".%s" % '/root/')
except tarfile.TarError as exc:
raise InfrastructureError(
"Unable to create lava overlay tarball: %s" % exc)
self.set_namespace_data(action=self.name,
label='output',
key='file',
value=output)
return connection
def run(self, connection, max_end_time, args=None):
if not self.parameters.get('ramdisk', None): # idempotency
return connection
ramdisk = self.get_namespace_data(action='download-action',
label='ramdisk',
key='file')
if self.skip:
self.logger.info("Not extracting ramdisk.")
suffix = self.get_namespace_data(action='tftp-deploy',
label='tftp',
key='suffix')
filename = os.path.join(suffix, "ramdisk",
os.path.basename(ramdisk))
# declare the original ramdisk as the name to be used later.
self.set_namespace_data(action='compress-ramdisk',
label='file',
key='ramdisk',
value=filename)
return
connection = super(ExtractRamdisk, self).run(connection, max_end_time,
args)
ramdisk_dir = self.mkdtemp()
extracted_ramdisk = os.path.join(ramdisk_dir, 'ramdisk')
os.mkdir(extracted_ramdisk, 0o755)
compression = self.parameters['ramdisk'].get('compression', None)
suffix = ""
if compression:
suffix = ".%s" % compression
ramdisk_compressed_data = os.path.join(ramdisk_dir,
RAMDISK_FNAME + suffix)
if self.parameters['ramdisk'].get('header', None) == 'u-boot':
cmd = ('dd if=%s of=%s ibs=%s skip=1' %
(ramdisk, ramdisk_compressed_data,
UBOOT_DEFAULT_HEADER_LENGTH)).split(' ')
try:
self.run_command(cmd)
except Exception:
raise LAVABug('Unable to remove uboot header: %s' % ramdisk)
else:
# give the file a predictable name
shutil.move(ramdisk, ramdisk_compressed_data)
ramdisk_data = decompress_file(ramdisk_compressed_data, compression)
with chdir(extracted_ramdisk):
cmd = ('cpio -iud -F %s' % ramdisk_data).split(' ')
if not self.run_command(cmd):
raise JobError(
'Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?'
% ramdisk_data)
# tell other actions where the unpacked ramdisk can be found
self.set_namespace_data(action=self.name,
label='extracted_ramdisk',
key='directory',
value=extracted_ramdisk)
self.set_namespace_data(action=self.name,
label='ramdisk_file',
key='file',
value=ramdisk_data)
return connection
def run(self, connection, max_end_time):
if not self.parameters.get("ramdisk"): # idempotency
return connection
ramdisk = self.get_namespace_data(
action="download-action", label="ramdisk", key="file"
)
if self.skip:
self.logger.info("Not extracting ramdisk.")
suffix = self.get_namespace_data(
action="tftp-deploy", label="tftp", key="suffix"
)
filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk))
# declare the original ramdisk as the name to be used later.
self.set_namespace_data(
action="compress-ramdisk", label="file", key="ramdisk", value=filename
)
return connection
connection = super().run(connection, max_end_time)
ramdisk_dir = self.mkdtemp()
extracted_ramdisk = os.path.join(ramdisk_dir, "ramdisk")
os.mkdir(extracted_ramdisk, 0o755)
compression = self.parameters["ramdisk"].get("compression")
suffix = ""
if compression:
suffix = ".%s" % compression
ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix)
if self.parameters["ramdisk"].get("header") == "u-boot":
cmd = (
"dd if=%s of=%s ibs=%s skip=1"
% (ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH)
).split(" ")
try:
self.run_command(cmd)
except Exception:
raise LAVABug("Unable to remove uboot header: %s" % ramdisk)
else:
# give the file a predictable name
shutil.move(ramdisk, ramdisk_compressed_data)
ramdisk_data = decompress_file(ramdisk_compressed_data, compression)
with chdir(extracted_ramdisk):
cmd = ("cpio -iud -F %s" % ramdisk_data).split(" ")
if not self.run_command(cmd):
raise JobError(
"Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?"
% ramdisk_data
)
# tell other actions where the unpacked ramdisk can be found
self.set_namespace_data(
action=self.name,
label="extracted_ramdisk",
key="directory",
value=extracted_ramdisk,
)
self.set_namespace_data(
action=self.name, label="ramdisk_file", key="file", value=ramdisk_data
)
return connection
def cpio(directory, filename):
which("cpio")
which("find")
with chdir(directory):
try:
find = subprocess.check_output(["find", "."],
stderr=subprocess.STDOUT) # nosec
return subprocess.check_output( # nosec
["cpio", "--create", "--format", "newc", "--file", filename],
input=find,
stderr=subprocess.STDOUT,
).decode("utf-8", errors="replace")
except Exception as exc:
raise InfrastructureError("Unable to create cpio archive %r: %s" %
(filename, exc))
def compress_file(infile, compression):
if not compression:
return infile
if compression not in compress_command_map.keys():
raise JobError("Cannot find shell command to compress: %s" % compression)
# Check that the command does exists
which(compress_command_map[compression][0])
with chdir(os.path.dirname(infile)):
# local copy for idempotency
cmd = compress_command_map[compression][:]
cmd.append(infile)
try:
subprocess.check_output(cmd) # nosec - internal use.
return "%s.%s" % (infile, compression)
except (OSError, subprocess.CalledProcessError) as exc:
raise InfrastructureError("unable to compress file %s: %s" % (infile, exc))
def uncpio(filename, directory):
which("cpio")
with chdir(directory):
try:
subprocess.check_output( # nosec
[
"cpio",
"--extract",
"--make-directories",
"--unconditional",
"--file",
filename,
],
stderr=subprocess.STDOUT,
)
except subprocess.SubprocessError as exc:
raise InfrastructureError("Unable to extract cpio archive %r: %s" %
(filename, exc))
def run(self, connection, max_end_time):
output = os.path.join(self.mkdtemp(), "overlay-%s.tar.gz" % self.level)
location = self.get_namespace_data(action="test",
label="shared",
key="location")
lava_test_results_dir = self.get_namespace_data(
action="test", label="results", key="lava_test_results_dir")
self.set_namespace_data(action="test",
label="shared",
key="output",
value=output)
if not location:
raise LAVABug("Missing lava overlay location")
if not os.path.exists(location):
raise LAVABug("Unable to find overlay location")
if not self.valid:
self.logger.error(self.errors)
return connection
connection = super().run(connection, max_end_time)
# This will split the path into two parts. Second part is only the
# bottom level directory. The first part is the rest.
# Example: /run/mount/lava/ will be split to /run/mount/ and lava/
# Used to create a tarball with only the bottom level dir included and
# not with the whole lava_test_results_dir property.
results_dir_list = os.path.split(
os.path.normpath(lava_test_results_dir))
with chdir(
os.path.join(location,
os.path.relpath(results_dir_list[0], os.sep))):
try:
with tarfile.open(output, "w:gz") as tar:
tar.add("%s" % results_dir_list[1])
# ssh authorization support
if os.path.exists("./root/"):
tar.add(".%s" % "/root/")
except tarfile.TarError as exc:
raise InfrastructureError(
"Unable to create lava overlay tarball: %s" % exc)
self.set_namespace_data(action=self.name,
label="output",
key="file",
value=output)
return connection
def compress_file(infile, compression):
if not compression:
return infile
if compression not in compress_command_map.keys():
raise JobError("Cannot find shell command to compress: %s" %
compression)
# Check that the command does exists
which(compress_command_map[compression].split(" ")[0])
with chdir(os.path.dirname(infile)):
cmd = "%s %s" % (compress_command_map[compression], infile)
try:
# safe to use shell=True here, no external arguments
subprocess.check_output(cmd, shell=True)
return "%s.%s" % (infile, compression)
except (OSError, subprocess.CalledProcessError) as exc:
raise InfrastructureError('unable to compress file %s: %s' %
(infile, exc))
def run(self, connection, max_end_time, args=None):
if not self.parameters.get('ramdisk', None): # idempotency
return connection
ramdisk = self.get_namespace_data(action='download-action', label='ramdisk', key='file')
if self.skip:
self.logger.info("Not extracting ramdisk.")
suffix = self.get_namespace_data(action='tftp-deploy', label='tftp', key='suffix')
filename = os.path.join(suffix, "ramdisk", os.path.basename(ramdisk))
# declare the original ramdisk as the name to be used later.
self.set_namespace_data(action='compress-ramdisk', label='file', key='ramdisk', value=filename)
return
connection = super(ExtractRamdisk, self).run(connection, max_end_time, args)
ramdisk_dir = self.mkdtemp()
extracted_ramdisk = os.path.join(ramdisk_dir, 'ramdisk')
os.mkdir(extracted_ramdisk, 0o755)
compression = self.parameters['ramdisk'].get('compression', None)
suffix = ""
if compression:
suffix = ".%s" % compression
ramdisk_compressed_data = os.path.join(ramdisk_dir, RAMDISK_FNAME + suffix)
if self.parameters['ramdisk'].get('header', None) == 'u-boot':
cmd = ('dd if=%s of=%s ibs=%s skip=1' % (
ramdisk, ramdisk_compressed_data, UBOOT_DEFAULT_HEADER_LENGTH)).split(' ')
try:
self.run_command(cmd)
except Exception:
raise LAVABug('Unable to remove uboot header: %s' % ramdisk)
else:
# give the file a predictable name
shutil.move(ramdisk, ramdisk_compressed_data)
ramdisk_data = decompress_file(ramdisk_compressed_data, compression)
with chdir(extracted_ramdisk):
cmd = ('cpio -iud -F %s' % ramdisk_data).split(' ')
if not self.run_command(cmd):
raise JobError('Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?' % ramdisk_data)
# tell other actions where the unpacked ramdisk can be found
self.set_namespace_data(action=self.name, label='extracted_ramdisk', key='directory', value=extracted_ramdisk)
self.set_namespace_data(action=self.name, label='ramdisk_file', key='file', value=ramdisk_data)
return connection
def run(self, connection, max_end_time, args=None): # pylint: disable=too-many-locals
if not self.parameters.get('ramdisk', None): # idempotency
return connection
if self.skip:
return connection
connection = super(CompressRamdisk, self).run(connection, max_end_time,
args)
ramdisk_dir = self.get_namespace_data(action='extract-overlay-ramdisk',
label='extracted_ramdisk',
key='directory')
ramdisk_data = self.get_namespace_data(
action='extract-overlay-ramdisk', label='ramdisk_file', key='file')
if not ramdisk_dir:
raise LAVABug("Unable to find unpacked ramdisk")
if not ramdisk_data:
raise LAVABug("Unable to find ramdisk directory")
if self.parameters.get('preseed', None):
if self.parameters["deployment_data"].get("preseed_to_ramdisk",
None):
# download action must have completed to get this far
# some installers (centos) cannot fetch the preseed file via tftp.
# Instead, put the preseed file into the ramdisk using a given name
# from deployment_data which we can use in the boot commands.
filename = self.parameters["deployment_data"][
"preseed_to_ramdisk"]
self.logger.info("Copying preseed file into ramdisk: %s",
filename)
shutil.copy(
self.get_namespace_data(action='download-action',
label='preseed',
key='file'),
os.path.join(ramdisk_dir, filename))
self.set_namespace_data(action=self.name,
label='file',
key='preseed_local',
value=filename)
with chdir(ramdisk_dir):
self.logger.info("Building ramdisk %s containing %s", ramdisk_data,
ramdisk_dir)
cmd = "find . | cpio --create --format='newc' > %s" % ramdisk_data
try:
# safe to use shell=True here, no external arguments
log = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
except OSError as exc:
raise InfrastructureError(
'Unable to create cpio filesystem: %s' % exc)
# lazy-logging would mean that the quoting of cmd causes invalid YAML
self.logger.debug("%s\n%s" % (cmd, log)) # pylint: disable=logging-not-lazy
# we need to compress the ramdisk with the same method is was submitted with
compression = self.parameters['ramdisk'].get('compression', None)
final_file = compress_file(ramdisk_data, compression)
tftp_dir = os.path.dirname(
self.get_namespace_data(action='download-action',
label='ramdisk',
key='file'))
if self.add_header == 'u-boot':
ramdisk_uboot = final_file + ".uboot"
self.logger.debug("Adding RAMdisk u-boot header.")
cmd = ("mkimage -A %s -T ramdisk -C none -d %s %s" %
(self.mkimage_arch, final_file, ramdisk_uboot)).split(' ')
if not self.run_command(cmd):
raise InfrastructureError(
"Unable to add uboot header to ramdisk")
final_file = ramdisk_uboot
full_path = os.path.join(tftp_dir, os.path.basename(final_file))
shutil.move(final_file, full_path)
self.logger.debug("rename %s to %s", final_file, full_path)
self.set_namespace_data(action=self.name,
label='file',
key='full-path',
value=full_path)
if self.parameters['to'] == 'tftp':
suffix = self.get_namespace_data(action='tftp-deploy',
label='tftp',
key='suffix')
if not suffix:
suffix = ''
self.set_namespace_data(action=self.name,
label='file',
key='ramdisk',
value=os.path.join(
suffix, "ramdisk",
os.path.basename(final_file)))
else:
self.set_namespace_data(action=self.name,
label='file',
key='ramdisk',
value=final_file)
return connection
def run(self, connection, max_end_time):
if not self.parameters.get("ramdisk"): # idempotency
return connection
if self.skip:
return connection
connection = super().run(connection, max_end_time)
ramdisk_dir = self.get_namespace_data(action="extract-overlay-ramdisk",
label="extracted_ramdisk",
key="directory")
ramdisk_data = self.get_namespace_data(
action="extract-overlay-ramdisk", label="ramdisk_file", key="file")
if not ramdisk_dir:
raise LAVABug("Unable to find unpacked ramdisk")
if not ramdisk_data:
raise LAVABug("Unable to find ramdisk directory")
if self.parameters.get("preseed"):
if self.parameters["deployment_data"].get("preseed_to_ramdisk"):
# download action must have completed to get this far
# some installers (centos) cannot fetch the preseed file via tftp.
# Instead, put the preseed file into the ramdisk using a given name
# from deployment_data which we can use in the boot commands.
filename = self.parameters["deployment_data"][
"preseed_to_ramdisk"]
self.logger.info("Copying preseed file into ramdisk: %s",
filename)
shutil.copy(
self.get_namespace_data(action="download-action",
label="preseed",
key="file"),
os.path.join(ramdisk_dir, filename),
)
self.set_namespace_data(action=self.name,
label="file",
key="preseed_local",
value=filename)
with chdir(ramdisk_dir):
self.logger.info("Building ramdisk %s containing %s", ramdisk_data,
ramdisk_dir)
cmd = "find . | cpio --create --format='newc' > %s" % ramdisk_data
try:
log = subprocess.check_output( # nosec - safe to use shell=True here, no external arguments
cmd,
shell=True,
stderr=subprocess.STDOUT)
log = log.decode("utf-8", errors="replace")
except OSError as exc:
raise InfrastructureError(
"Unable to create cpio filesystem: %s" % exc)
# lazy-logging would mean that the quoting of cmd causes invalid YAML
self.logger.debug("%s\n%s" % (cmd, log))
# we need to compress the ramdisk with the same method is was submitted with
compression = self.parameters["ramdisk"].get("compression")
final_file = compress_file(ramdisk_data, compression)
tftp_dir = os.path.dirname(
self.get_namespace_data(action="download-action",
label="ramdisk",
key="file"))
if self.add_header == "u-boot":
ramdisk_uboot = final_file + ".uboot"
self.logger.debug("Adding RAMdisk u-boot header.")
cmd = ("mkimage -A %s -T ramdisk -C none -d %s %s" %
(self.mkimage_arch, final_file, ramdisk_uboot)).split(" ")
if not self.run_command(cmd):
raise InfrastructureError(
"Unable to add uboot header to ramdisk")
final_file = ramdisk_uboot
full_path = os.path.join(tftp_dir, os.path.basename(final_file))
shutil.move(final_file, full_path)
self.logger.debug("rename %s to %s", final_file, full_path)
self.set_namespace_data(action=self.name,
label="file",
key="full-path",
value=full_path)
if self.parameters["to"] == "tftp":
suffix = self.get_namespace_data(action="tftp-deploy",
label="tftp",
key="suffix")
if not suffix:
suffix = ""
self.set_namespace_data(
action=self.name,
label="file",
key="ramdisk",
value=os.path.join(suffix, "ramdisk",
os.path.basename(final_file)),
)
else:
self.set_namespace_data(action=self.name,
label="file",
key="ramdisk",
value=final_file)
return connection
def run(self, connection, max_end_time, args=None): # pylint: disable=too-many-locals
if not self.parameters.get('ramdisk', None): # idempotency
return connection
if self.skip:
return connection
connection = super(CompressRamdisk, self).run(connection, max_end_time, args)
ramdisk_dir = self.get_namespace_data(
action='extract-overlay-ramdisk', label='extracted_ramdisk', key='directory')
ramdisk_data = self.get_namespace_data(
action='extract-overlay-ramdisk', label='ramdisk_file', key='file')
if not ramdisk_dir:
raise LAVABug("Unable to find unpacked ramdisk")
if not ramdisk_data:
raise LAVABug("Unable to find ramdisk directory")
if self.parameters.get('preseed', None):
if self.parameters["deployment_data"].get("preseed_to_ramdisk", None):
# download action must have completed to get this far
# some installers (centos) cannot fetch the preseed file via tftp.
# Instead, put the preseed file into the ramdisk using a given name
# from deployment_data which we can use in the boot commands.
filename = self.parameters["deployment_data"]["preseed_to_ramdisk"]
self.logger.info("Copying preseed file into ramdisk: %s", filename)
shutil.copy(self.get_namespace_data(
action='download-action', label='preseed',
key='file'), os.path.join(ramdisk_dir, filename))
self.set_namespace_data(action=self.name, label='file', key='preseed_local', value=filename)
with chdir(ramdisk_dir):
self.logger.info("Building ramdisk %s containing %s",
ramdisk_data, ramdisk_dir)
cmd = "find . | cpio --create --format='newc' > %s" % ramdisk_data
try:
# safe to use shell=True here, no external arguments
log = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
except OSError as exc:
raise InfrastructureError('Unable to create cpio filesystem: %s' % exc)
# lazy-logging would mean that the quoting of cmd causes invalid YAML
self.logger.debug("%s\n%s" % (cmd, log)) # pylint: disable=logging-not-lazy
# we need to compress the ramdisk with the same method is was submitted with
compression = self.parameters['ramdisk'].get('compression', None)
final_file = compress_file(ramdisk_data, compression)
tftp_dir = os.path.dirname(self.get_namespace_data(
action='download-action', label='ramdisk', key='file'))
if self.add_header == 'u-boot':
ramdisk_uboot = final_file + ".uboot"
self.logger.debug("Adding RAMdisk u-boot header.")
cmd = ("mkimage -A %s -T ramdisk -C none -d %s %s" % (self.mkimage_arch, final_file, ramdisk_uboot)).split(' ')
if not self.run_command(cmd):
raise InfrastructureError("Unable to add uboot header to ramdisk")
final_file = ramdisk_uboot
full_path = os.path.join(tftp_dir, os.path.basename(final_file))
shutil.move(final_file, full_path)
self.logger.debug("rename %s to %s", final_file, full_path)
self.set_namespace_data(
action=self.name, label='file', key='full-path', value=full_path)
if self.parameters['to'] == 'tftp':
suffix = self.get_namespace_data(action='tftp-deploy', label='tftp', key='suffix')
if not suffix:
suffix = ''
self.set_namespace_data(
action=self.name,
label='file', key='ramdisk', value=os.path.join(suffix, "ramdisk", os.path.basename(final_file)))
else:
self.set_namespace_data(
action=self.name,
label='file', key='ramdisk', value=final_file)
return connection
