def _make_pipeline(self, params):
pipeline = Pipeline()
auto_login = AutoLoginAction()
auto_login.section = "internal"
auto_login.parameters = params
pipeline.add_action(auto_login)
return pipeline
class MountAction(DeployAction):
"""
Depending on the type of deployment, this needs to perform
an OffsetAction, LoopCheckAction, LoopMountAction
"""
name = "mount-action"
description = "mount with offset"
summary = "mount loop"
def __init__(self, key):
super(MountAction, self).__init__()
self.key = key
def populate(self, parameters):
"""
Needs to take account of the deployment type / image type etc.
to determine which actions need to be added to the internal pipeline
as part of the deployment selection step.
"""
if not self.job:
raise LAVABug("No job object supplied to action")
# FIXME: not all mount operations will need these actions
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(OffsetAction(self.key))
# FIXME: LoopCheckAction and LoopMountAction should be in only one Action
self.internal_pipeline.add_action(LoopCheckAction(self.key))
self.internal_pipeline.add_action(LoopMountAction(self.key))
class BootCMSISRetry(RetryAction):
name = "boot-cmsis-retry"
description = "boot cmsis usb image with retry"
summary = "boot cmsis usb image with retry"
def validate(self):
super(BootCMSISRetry, self).validate()
method_params = self.job.device['actions']['boot']['methods']['cmsis-dap']['parameters']
usb_mass_device = method_params.get('usb_mass_device', None)
if not usb_mass_device:
self.errors = "usb_mass_device unset"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
method_params = self.job.device['actions']['boot']['methods']['cmsis-dap']['parameters']
usb_mass_device = method_params.get('usb_mass_device', None)
resets_after_flash = method_params.get('resets_after_flash', True)
if self.job.device.hard_reset_command:
self.internal_pipeline.add_action(ResetDevice())
self.internal_pipeline.add_action(WaitDevicePathAction(usb_mass_device))
self.internal_pipeline.add_action(FlashCMSISAction())
if resets_after_flash:
self.internal_pipeline.add_action(WaitUSBSerialDeviceAction())
self.internal_pipeline.add_action(ConnectDevice())
class UnmountAction(RetryAction):
name = "umount-retry"
description = "retry support for umount"
summary = "retry umount"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(Unmount())
class BootDockerRetry(RetryAction):
name = 'boot-docker-retry'
description = "boot docker image with retry"
summary = "boot docker image"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(CallDockerAction())
class BootCMSIS(BootAction):
name = "boot-cmsis"
description = "boot cmsis usb image"
summary = "boot cmsis usb image"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(BootCMSISRetry())
class BootDFU(BootAction):
name = 'boot-dfu-image'
description = "boot dfu image with retry"
summary = "boot dfu image with retry"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(BootDFURetry())
class TestShellRetry(RetryAction):
name = "lava-test-retry"
description = "Retry wrapper for lava-test-shell"
summary = "Retry support for Lava Test Shell"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(TestShellAction())
class BootQemuRetry(RetryAction):
name = 'boot-qemu-image'
description = "boot image using QEMU command line"
summary = "boot QEMU image"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(CallQemuAction())
class InternalRetryAction(RetryAction):
section = 'internal'
name = "internal-retry-action"
description = "internal, do not use outside unit tests"
summary = "internal retry action for unit tests"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job)
self.internal_pipeline.add_action(TestAction.FakeAction(), parameters)
def test_overlay_action(self): # pylint: disable=too-many-locals
parameters = {
'device_type': 'd02',
'job_name': 'grub-standard-ramdisk',
'job_timeout': '15m',
'action_timeout': '5m',
'priority': 'medium',
'actions': {
'boot': {
'method': 'grub',
'commands': 'ramdisk',
'prompts': ['linaro-test', 'root@debian:~#']
},
'deploy': {
'ramdisk': 'initrd.gz',
'kernel': 'zImage',
'dtb': 'broken.dtb'
}
}
}
device = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/d02-01.yaml'))
job = Job(4212, parameters, None)
job.device = device
pipeline = Pipeline(job=job, parameters=parameters['actions']['boot'])
job.pipeline = pipeline
overlay = BootloaderCommandOverlay()
pipeline.add_action(overlay)
ip_addr = dispatcher_ip(None)
parsed = []
kernel = parameters['actions']['deploy']['kernel']
ramdisk = parameters['actions']['deploy']['ramdisk']
dtb = parameters['actions']['deploy']['dtb']
substitution_dictionary = {
'{SERVER_IP}': ip_addr,
# the addresses need to be hexadecimal
'{RAMDISK}': ramdisk,
'{KERNEL}': kernel,
'{DTB}': dtb
}
params = device['actions']['boot']['methods']
commands = params['grub']['ramdisk']['commands']
self.assertIn('net_bootp', commands)
self.assertIn("linux (tftp,{SERVER_IP})/{KERNEL} console=ttyS0,115200 earlycon=uart8250,mmio32,0x80300000 root=/dev/ram0 ip=dhcp", commands)
self.assertIn('initrd (tftp,{SERVER_IP})/{RAMDISK}', commands)
self.assertIn('devicetree (tftp,{SERVER_IP})/{DTB}', commands)
params['grub']['ramdisk']['commands'] = substitute(params['grub']['ramdisk']['commands'], substitution_dictionary)
substituted_commands = params['grub']['ramdisk']['commands']
self.assertIs(type(substituted_commands), list)
self.assertIn('net_bootp', substituted_commands)
self.assertNotIn("linux (tftp,{SERVER_IP})/{KERNEL} console=ttyS0,115200 earlycon=uart8250,mmio32,0x80300000 root=/dev/ram0 ip=dhcp", substituted_commands)
self.assertIn("linux (tftp,%s)/%s console=ttyS0,115200 earlycon=uart8250,mmio32,0x80300000 root=/dev/ram0 ip=dhcp" % (ip_addr, kernel), substituted_commands)
self.assertNotIn('initrd (tftp,{SERVER_IP})/{RAMDISK}', parsed)
self.assertNotIn('devicetree (tftp,{SERVER_IP})/{DTB}', parsed)
class BootQEMUImageAction(BootAction):
name = 'boot-image-retry'
description = "boot image with retry"
summary = "boot with retry"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(BootQemuRetry())
if self.has_prompts(parameters):
self.internal_pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.internal_pipeline.add_action(ExpectShellSession())
if 'transfer_overlay' in parameters:
self.internal_pipeline.add_action(OverlayUnpack())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
if self.job.device.hard_reset_command:
self.internal_pipeline.add_action(ResetDevice())
self.internal_pipeline.add_action(WaitDeviceBoardID(self.job.device.get('board_id', None)))
self.internal_pipeline.add_action(FlashPyOCDAction())
self.internal_pipeline.add_action(ConnectDevice())
def populate(self, parameters):
"""
The dispatcher does the first download as the first deployment is not guaranteed to
have DNS resolution fully working, so we can use the IP address of the dispatcher
to get it (with the advantage that the dispatcher decompresses it so that the ramdisk
can pipe the raw image directly from wget to dd.
This also allows the use of local file:// locations which are visible to the dispatcher
but not the device.
"""
self.image_path = self.mkdtemp()
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(OverlayAction()) # idempotent, includes testdef
uniquify = parameters.get('uniquify', True)
if 'images' in parameters:
for k in sorted(parameters['images'].keys()):
if k == 'yaml_line':
continue
self.internal_pipeline.add_action(DownloaderAction(
k, path=self.image_path, uniquify=uniquify))
if parameters['images'][k].get('apply-overlay', False):
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(ApplyOverlayImage())
self.internal_pipeline.add_action(DDAction())
elif 'image' in parameters:
self.internal_pipeline.add_action(DownloaderAction(
'image', path=self.image_path, uniquify=uniquify))
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(ApplyOverlayImage())
self.internal_pipeline.add_action(DDAction())
# FIXME: could support tarballs too
if self.test_needs_deployment(parameters):
self.internal_pipeline.add_action(DeployDeviceEnvironment())
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(OverlayAction())
# Check if the device has a power command such as HiKey, Dragonboard,
# etc. against device that doesn't like Nexus, etc.
if self.job.device.get('fastboot_via_uboot', False):
self.internal_pipeline.add_action(ConnectDevice())
self.internal_pipeline.add_action(UBootEnterFastbootAction())
elif self.job.device.hard_reset_command:
self.force_prompt = True
self.internal_pipeline.add_action(ConnectDevice())
self.internal_pipeline.add_action(ResetDevice())
else:
self.internal_pipeline.add_action(EnterFastbootAction())
fastboot_dir = self.mkdtemp()
image_keys = sorted(parameters['images'].keys())
for image in image_keys:
if image != 'yaml_line':
self.internal_pipeline.add_action(DownloaderAction(image, fastboot_dir))
if parameters['images'][image].get('apply-overlay', False):
if self.test_needs_overlay(parameters):
if parameters['images'][image].get('sparse', True):
self.internal_pipeline.add_action(
ApplyOverlaySparseImage(image))
else:
self.internal_pipeline.add_action(
ApplyOverlayImage(image, use_root_partition=False))
if self.test_needs_overlay(parameters) and \
self.test_needs_deployment(parameters):
self.internal_pipeline.add_action(
DeployDeviceEnvironment())
self.internal_pipeline.add_action(FastbootFlashOrderAction())
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
# customize the device configuration for this job
self.internal_pipeline.add_action(UBootSecondaryMedia())
self.internal_pipeline.add_action(BootloaderCommandOverlay())
self.internal_pipeline.add_action(ConnectDevice())
self.internal_pipeline.add_action(UBootRetry())
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(BootDockerRetry())
if self.has_prompts(parameters):
if self.test_has_shell(parameters):
self.internal_pipeline.add_action(ExpectShellSession())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job,
parameters=parameters)
# Check if the device has a power command such as HiKey, Dragonboard,
# etc. against device that doesn't like Nexus, etc.
# This is required in order to power on the device so that when the
# test job writer wants to perform some operation using a
# lava-test-shell action that follows, this becomes mandatory. Think of
# issuing any fastboot commands on the powered on device.
#
# NOTE: Add more power on strategies, if required for specific devices.
if self.job.device.get('fastboot_via_uboot', False):
self.internal_pipeline.add_action(ConnectDevice())
self.internal_pipeline.add_action(UBootEnterFastbootAction())
elif self.job.device.hard_reset_command:
self.force_prompt = True
self.internal_pipeline.add_action(ConnectDevice())
self.internal_pipeline.add_action(ResetDevice())
else:
self.internal_pipeline.add_action(EnterFastbootAction())
self.download_dir = self.mkdtemp()
image_keys = sorted(parameters['images'].keys())
for image in image_keys:
if image != 'yaml_line':
self.internal_pipeline.add_action(DownloaderAction(
image, self.download_dir))
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(OverlayAction())
self.internal_pipeline.add_action(CopyToLxcAction())
class ResetDevice(Action):
"""
Used within a RetryAction - first tries 'reboot' then
tries PDU.
"""
name = "reset-device"
description = "reboot or power-cycle the device"
summary = "reboot the device"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
if self.job.device.hard_reset_command:
self.internal_pipeline.add_action(PDUReboot())
else:
self.internal_pipeline.add_action(SendRebootCommands())
def populate(self, parameters):
super(GrubSequenceAction, self).populate(parameters)
self.internal_pipeline = Pipeline(parent=self, job=self.job,
parameters=parameters)
sequences = self.job.device['actions']['boot']['methods']['grub'].get(
'sequence', [])
for sequence in sequences:
mapped = _grub_sequence_map(sequence)
if mapped[1]:
self.internal_pipeline.add_action(
mapped[0](type=mapped[1]))
elif mapped[0]:
self.internal_pipeline.add_action(mapped[0]())
if self.has_prompts(parameters):
self.internal_pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.internal_pipeline.add_action(ExpectShellSession())
if 'transfer_overlay' in parameters:
self.internal_pipeline.add_action(OverlayUnpack())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
else:
if self.has_boot_finished(parameters):
self.logger.debug("Doing a boot without a shell (installer)")
self.internal_pipeline.add_action(InstallerWait())
self.internal_pipeline.add_action(PowerOff())
def populate(self, parameters):
self.expect_shell = parameters.get('expect_shell', True)
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(BootloaderSecondaryMedia())
self.internal_pipeline.add_action(BootloaderCommandOverlay())
self.internal_pipeline.add_action(ConnectDevice())
# FIXME: reset_device is a hikey hack due to fastboot/OTG issues
# remove as part of LAVA-940 - convert to use fastboot-sequence
reset_device = self.job.device['actions']['boot']['methods'].get('grub-efi', {}).get('reset_device', True)
if parameters['method'] == 'grub-efi' and reset_device:
# added unless the device specifies not to reset the device in grub.
self.internal_pipeline.add_action(ResetDevice())
elif parameters['method'] == 'grub':
self.internal_pipeline.add_action(ResetDevice())
if parameters['method'] == 'grub-efi':
self.internal_pipeline.add_action(UEFIMenuInterrupt())
self.internal_pipeline.add_action(GrubMenuSelector())
self.internal_pipeline.add_action(BootloaderInterruptAction())
self.internal_pipeline.add_action(BootloaderCommandsAction())
if self.has_prompts(parameters):
self.internal_pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.internal_pipeline.add_action(ExpectShellSession())
if 'transfer_overlay' in parameters:
self.internal_pipeline.add_action(OverlayUnpack())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
else:
if self.has_boot_finished(parameters):
self.logger.debug("Doing a boot without a shell (installer)")
self.internal_pipeline.add_action(InstallerWait())
self.internal_pipeline.add_action(PowerOff())
def test_overlay_action(self): # pylint: disable=too-many-locals
parameters = {
'device_type': 'x86',
'job_name': 'ipxe-pipeline',
'job_timeout': '15m',
'action_timeout': '5m',
'priority': 'medium',
'actions': {
'boot': {
'method': 'ipxe',
'commands': 'ramdisk',
'prompts': ['linaro-test', 'root@debian:~#']
},
'deploy': {
'ramdisk': 'initrd.gz',
'kernel': 'zImage',
}
}
}
device = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/x86-01.yaml'))
job = Job(4212, parameters, None)
job.device = device
pipeline = Pipeline(job=job, parameters=parameters['actions']['boot'])
job.pipeline = pipeline
overlay = BootloaderCommandOverlay()
pipeline.add_action(overlay)
ip_addr = dispatcher_ip(None)
kernel = parameters['actions']['deploy']['kernel']
ramdisk = parameters['actions']['deploy']['ramdisk']
substitution_dictionary = {
'{SERVER_IP}': ip_addr,
'{RAMDISK}': ramdisk,
'{KERNEL}': kernel,
'{LAVA_MAC}': "00:00:00:00:00:00"
}
params = device['actions']['boot']['methods']
params['ipxe']['ramdisk']['commands'] = substitute(params['ipxe']['ramdisk']['commands'], substitution_dictionary)
commands = params['ipxe']['ramdisk']['commands']
self.assertIs(type(commands), list)
self.assertIn("dhcp net0", commands)
self.assertIn("set console console=ttyS0,115200n8 lava_mac=00:00:00:00:00:00", commands)
self.assertIn("set extraargs init=/sbin/init ip=dhcp", commands)
self.assertNotIn("kernel tftp://{SERVER_IP}/{KERNEL} ${extraargs} ${console}", commands)
self.assertNotIn("initrd tftp://{SERVER_IP}/{RAMDISK}", commands)
self.assertIn("boot", commands)
class DockerAction(DeployAction):
name = "deploy-docker"
description = "deploy docker images"
summary = "deploy docker"
def validate(self):
super(DockerAction, self).validate()
which("docker")
# Print docker version
try:
out = subprocess.check_output(["docker", "version", "-f", "{{.Server.Version}}"])
out = out.decode("utf-8", errors="replace").strip("\n")
self.logger.debug("docker server, installed at version: %s", out)
out = subprocess.check_output(["docker", "version", "-f", "{{.Client.Version}}"])
out = out.decode("utf-8", errors="replace").strip("\n")
self.logger.debug("docker client, installed at version: %s", out)
except subprocess.CalledProcessError as exc:
raise InfrastructureError("Unable to call '%s': %s" % (exc.cmd, exc.output))
except OSError:
raise InfrastructureError("Command 'docker' does not exist")
# check docker image name
# The string should be safe for command line inclusion
image_name = self.parameters["image"]
if re.compile("^[a-z0-9._:/-]+$").match(image_name) is None:
self.errors = "image_name '%s' is invalid" % image_name
self.set_namespace_data(action=self.name, label='image',
key='name', value=image_name)
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
if self.test_needs_deployment(parameters):
self.internal_pipeline.add_action(DeployDeviceEnvironment())
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(OverlayAction())
def run(self, connection, max_end_time, args=None):
# Pull the image
cmd = ["docker", "pull", self.parameters["image"]]
out = self.run_command(cmd, allow_fail=False, allow_silent=False)
if not out:
msg = "Unable to pull docker image '%s'" % self.parameters["image"]
raise JobError(msg)
return super(DockerAction, self).run(connection, max_end_time, args)
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job,
parameters=parameters)
# establish a new connection before trying the reset
self.internal_pipeline.add_action(ResetDevice())
# need to look for Hit any key to stop autoboot
self.internal_pipeline.add_action(BootloaderInterruptAction())
self.internal_pipeline.add_action(ConnectLxc())
class SecondaryShellAction(BootAction):
name = "secondary-shell-action"
description = "Connect to a secondary shell on specified hardware"
summary = "connect to a specified second shell"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
name = parameters['connection']
self.internal_pipeline.add_action(ConnectShell(name=name))
if self.has_prompts(parameters):
self.internal_pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.internal_pipeline.add_action(ExpectShellSession())
if 'transfer_overlay' in parameters:
self.internal_pipeline.add_action(OverlayUnpack())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
# the logic here can be upgraded in future if needed with more parameters to the deploy.
methods = self.job.device['actions']['boot']['methods']
if 'u-boot' in methods:
self.internal_pipeline.add_action(UBootPrepareKernelAction())
elif 'depthcharge' in methods:
self.internal_pipeline.add_action(PrepareFITAction())
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(KexecAction())
# Add AutoLoginAction unconditionally as this action does nothing if
# the configuration does not contain 'auto_login'
self.internal_pipeline.add_action(AutoLoginAction())
self.internal_pipeline.add_action(ExpectShellSession())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
class PrepareKernelAction(Action):
"""
Populate the pipeline with a kernel conversion action, if needed
"""
name = "prepare-kernel"
description = "populates the pipeline with a kernel conversion action"
summary = "add a kernel conversion"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
# the logic here can be upgraded in future if needed with more parameters to the deploy.
methods = self.job.device['actions']['boot']['methods']
if 'u-boot' in methods:
self.internal_pipeline.add_action(UBootPrepareKernelAction())
elif 'depthcharge' in methods:
self.internal_pipeline.add_action(PrepareFITAction())
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(LxcStartAction())
self.internal_pipeline.add_action(LxcAddStaticDevices())
self.internal_pipeline.add_action(ConnectLxc())
# Skip AutoLoginAction unconditionally as this action tries to parse kernel message
# self.internal_pipeline.add_action(AutoLoginAction())
self.internal_pipeline.add_action(ExpectShellSession())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
class BootLxcAction(BootAction):
"""
Provide for auto_login parameters in this boot stanza and re-establish the
connection after boot.
"""
name = "lxc-boot"
description = "lxc boot into the system"
summary = "lxc boot"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
self.internal_pipeline.add_action(LxcStartAction())
self.internal_pipeline.add_action(LxcAddStaticDevices())
self.internal_pipeline.add_action(ConnectLxc())
# Skip AutoLoginAction unconditionally as this action tries to parse kernel message
# self.internal_pipeline.add_action(AutoLoginAction())
self.internal_pipeline.add_action(ExpectShellSession())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
class TestDefinitionAction(Action):
name = "test-definition"
description = "load test definitions into image"
summary = "loading test definitions"
def __init__(self):
"""
The TestDefinitionAction installs each test definition into
the overlay. It does not execute the scripts in the test
definition, that is the job of the TestAction class.
One TestDefinitionAction handles all test definitions for
the current job.
In addition, a TestOverlayAction is added to the pipeline
to handle parts of the overlay which are test definition dependent.
"""
super().__init__()
self.test_list = None
self.stages = 0
self.run_levels = {}
def populate(self, parameters):
"""
Each time a test definition is processed by a handler, a new set of
overlay files are needed, based on that test definition. Basic overlay
files are created by TestOverlayAction. More complex scripts like the
install:deps script and the main run script have custom Actions.
"""
index = []
self.pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.test_list = identify_test_definitions(self.job.test_info,
parameters["namespace"])
if self.test_list:
self.set_namespace_data(
action=self.name,
label=self.name,
key="test_list",
value=self.test_list,
parameters=parameters,
)
for testdefs in self.test_list:
for testdef in testdefs:
# namespace support allows only running the install steps for the relevant
# deployment as the next deployment could be a different OS.
handler = RepoAction.select(testdef["from"])()
# set the full set of job YAML parameters for this handler as handler parameters.
handler.job = self.job
handler.parameters = testdef
# store the correct test_name before appending to the local index
handler.parameters["test_name"] = "%s_%s" % (
len(index),
handler.parameters["name"],
)
self.pipeline.add_action(handler)
# a genuinely unique ID based on the *database* JobID and
# pipeline level for reproducibility and tracking -
# {DB-JobID}_{PipelineLevel}, e.g. 15432.0_3.5.4
handler.uuid = "%s_%s" % (self.job.job_id, handler.level)
handler.stage = self.stages
self.run_levels[testdef["name"]] = self.stages
# copy details into the overlay, one per handler but the same class each time.
overlay = TestOverlayAction()
overlay.job = self.job
overlay.parameters = testdef
overlay.parameters["test_name"] = handler.parameters[
"test_name"]
overlay.test_uuid = handler.uuid
# add install handler - uses job parameters
installer = TestInstallAction()
installer.job = self.job
installer.parameters = testdef
installer.parameters["test_name"] = handler.parameters[
"test_name"]
installer.test_uuid = handler.uuid
# add runsh handler - uses job parameters
runsh = TestRunnerAction()
runsh.job = self.job
runsh.parameters = testdef
runsh.parameters["test_name"] = handler.parameters["test_name"]
runsh.test_uuid = handler.uuid
index.append(handler.parameters["name"])
# add overlay handlers to the pipeline
self.pipeline.add_action(overlay)
self.pipeline.add_action(installer)
self.pipeline.add_action(runsh)
self.set_namespace_data(
action="test-definition",
label="test-definition",
key="testdef_index",
value=index,
parameters=parameters,
)
self.stages += 1
def validate(self):
"""
TestDefinitionAction is part of the overlay and therefore part of the deployment -
the internal pipeline then looks inside the job definition for details of the tests to deploy.
Jobs with no test actions defined (empty test_list) are explicitly allowed.
"""
if not self.job:
self.errors = "missing job object"
return
if "actions" not in self.job.parameters:
self.errors = "No actions defined in job parameters"
return
if not self.test_list:
return
exp = re.compile(DEFAULT_TESTDEF_NAME_CLASS)
for testdefs in self.test_list:
for testdef in testdefs:
if "parameters" in testdef: # optional
if not isinstance(testdef["parameters"], dict):
self.errors = "Invalid test definition parameters"
if "from" not in testdef:
self.errors = "missing 'from' field in test definition %s" % testdef
if "name" not in testdef:
self.errors = "missing 'name' field in test definition %s" % testdef
else:
res = exp.match(testdef["name"])
if not res:
self.errors = (
"Invalid characters found in test definition name: %s"
% testdef["name"])
super().validate()
for testdefs in self.test_list:
for testdef in testdefs:
try:
RepoAction.select(testdef["from"])()
except JobError as exc:
self.errors = str(exc)
def run(self, connection, max_end_time):
"""
Creates the list of test definitions for this Test
:param connection: Connection object, if any.
:param max_end_time: remaining time before block timeout.
:return: the received Connection.
"""
location = self.get_namespace_data(action="test",
label="shared",
key="location")
lava_test_results_dir = self.get_namespace_data(
action="test", label="results", key="lava_test_results_dir")
if not location:
raise LAVABug("Missing lava overlay location")
if not os.path.exists(location):
raise LAVABug("Unable to find overlay location")
self.logger.info("Loading test definitions")
# overlay_path is the location of the files before boot
overlay_base = os.path.abspath("%s/%s" %
(location, lava_test_results_dir))
self.set_namespace_data(
action="test",
label="test-definition",
key="overlay_dir",
value=overlay_base,
)
connection = super().run(connection, max_end_time)
self.logger.info("Creating lava-test-runner.conf files")
for stage in range(self.stages):
path = "%s/%s" % (overlay_base, stage)
self.logger.debug("Using lava-test-runner path: %s for stage %d",
path, stage)
with open("%s/%s/lava-test-runner.conf" % (overlay_base, stage),
"a") as runner_conf:
for handler in self.pipeline.actions:
if isinstance(handler,
RepoAction) and handler.stage == stage:
self.logger.debug("- %s",
handler.parameters["test_name"])
runner_conf.write(handler.runner)
return connection
def test_overlay_action(self, which_mock):
parameters = {
"dispatcher":
{}, # fake dispatcher parameter. Normally added by parser
"device_type": "beaglebone-black",
"job_name": "uboot-pipeline",
"job_timeout": "15m",
"action_timeout": "5m",
"priority": "medium",
"actions": {
"boot": {
"namespace": "common",
"method": "u-boot",
"commands": "ramdisk",
"prompts": ["linaro-test", "root@debian:~#"],
},
"deploy": {
"namespace": "common",
"ramdisk": {
"url": "initrd.gz",
"compression": "gz"
},
"kernel": {
"url": "zImage",
"type": "zimage"
},
"dtb": {
"url": "broken.dtb"
},
},
},
}
data = yaml_safe_load(Factory().create_device("bbb-01.jinja2")[0])
device = NewDevice(data)
job = Job(4212, parameters, None)
job.device = device
pipeline = Pipeline(job=job, parameters=parameters["actions"]["boot"])
job.pipeline = pipeline
overlay = BootloaderCommandOverlay()
connection = MagicMock()
connection.timeout = MagicMock()
pipeline.add_action(overlay)
overlay.set_namespace_data(
action="uboot-prepare-kernel",
label="bootcommand",
key="bootcommand",
value="bootz",
)
overlay.validate()
overlay.run(connection, 100)
ip_addr = dispatcher_ip(None)
parsed = []
kernel_addr = job.device["parameters"][overlay.bootcommand]["ramdisk"]
ramdisk_addr = job.device["parameters"][overlay.bootcommand]["ramdisk"]
dtb_addr = job.device["parameters"][overlay.bootcommand]["dtb"]
kernel = parameters["actions"]["deploy"]["kernel"]["url"]
ramdisk = parameters["actions"]["deploy"]["ramdisk"]["url"]
dtb = parameters["actions"]["deploy"]["dtb"]["url"]
substitution_dictionary = {
"{SERVER_IP}":
ip_addr,
# the addresses need to be hexadecimal
"{KERNEL_ADDR}":
kernel_addr,
"{DTB_ADDR}":
dtb_addr,
"{RAMDISK_ADDR}":
ramdisk_addr,
"{BOOTX}":
"%s %s %s %s" %
(overlay.bootcommand, kernel_addr, ramdisk_addr, dtb_addr),
"{RAMDISK}":
ramdisk,
"{KERNEL}":
kernel,
"{DTB}":
dtb,
}
params = device["actions"]["boot"]["methods"]
params["u-boot"]["ramdisk"]["commands"] = substitute(
params["u-boot"]["ramdisk"]["commands"], substitution_dictionary)
commands = params["u-boot"]["ramdisk"]["commands"]
self.assertIs(type(commands), list)
self.assertIn("tftp 0x83000000 zImage", commands)
self.assertIn("tftp 0x83000000 initrd.gz", commands)
self.assertIn("setenv initrd_size ${filesize}", commands)
self.assertIn("tftp 0x88000000 broken.dtb", commands)
self.assertNotIn("setenv kernel_addr_r '{KERNEL_ADDR}'", commands)
self.assertNotIn("setenv initrd_addr_r '{RAMDISK_ADDR}'", commands)
self.assertNotIn("setenv fdt_addr_r '{DTB_ADDR}'", commands)
for line in params["u-boot"]["ramdisk"]["commands"]:
line = line.replace("{SERVER_IP}", ip_addr)
# the addresses need to be hexadecimal
line = line.replace("{KERNEL_ADDR}", kernel_addr)
line = line.replace("{DTB_ADDR}", dtb_addr)
line = line.replace("{RAMDISK_ADDR}", ramdisk_addr)
line = line.replace(
"{BOOTX}",
"%s %s %s %s" %
(overlay.bootcommand, kernel_addr, ramdisk_addr, dtb_addr),
)
line = line.replace("{RAMDISK}", ramdisk)
line = line.replace("{KERNEL}", kernel)
line = line.replace("{DTB}", dtb)
parsed.append(line)
self.assertNotIn("setenv kernel_addr_r '{KERNEL_ADDR}'", parsed)
self.assertNotIn("setenv initrd_addr_r '{RAMDISK_ADDR}'", parsed)
self.assertNotIn("setenv fdt_addr_r '{DTB_ADDR}'", parsed)
def test_create_pipeline(self):
action = Action()
action.name = "internal_pipe"
action.description = "test action only"
action.summary = "starter"
pipe = Pipeline()
pipe.add_action(action)
self.assertEqual(len(pipe.actions), 1)
self.assertEqual(action.level, "1")
action = Action()
action.name = "child_action"
action.summary = "child"
action.description = "action implementing an internal pipe"
with self.assertRaises(LAVABug):
Pipeline(action)
pipe.add_action(action)
self.assertEqual(action.level, "2")
self.assertEqual(len(pipe.actions), 2)
# a formal RetryAction would contain a pre-built pipeline which can be inserted directly
retry_pipe = Pipeline(action)
action = Action()
action.name = "inside_action"
action.description = "action inside the internal pipe"
action.summary = "child"
retry_pipe.add_action(action)
self.assertEqual(len(retry_pipe.actions), 1)
self.assertEqual(action.level, "2.1")
def test_change_connection(self):
pipe = Pipeline()
pipe.add_action(TestFakeActions.MakeNewConnection())
conn = object()
self.assertIsNot(conn, pipe.run_actions(conn, None))
def test_list_of_subcommands(self):
pipe = Pipeline()
pipe.add_action(self.sub0)
pipe.add_action(self.sub1)
self.assertIs(pipe.actions[0], self.sub0)
self.assertIs(pipe.actions[1], self.sub1)
class GrubMainAction(BootAction):
name = "grub-main-action"
description = "main grub boot action"
summary = "run grub boot from power to system"
def __init__(self):
super().__init__()
self.expect_shell = True
def populate(self, parameters):
self.expect_shell = parameters.get("expect_shell", True)
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.internal_pipeline.add_action(BootloaderSecondaryMedia())
self.internal_pipeline.add_action(BootloaderCommandOverlay())
self.internal_pipeline.add_action(ConnectDevice())
# FIXME: reset_device is a hikey hack due to fastboot/OTG issues
# remove as part of LAVA-940 - convert to use fastboot-sequence
reset_device = (self.job.device["actions"]["boot"]["methods"].get(
"grub-efi", {}).get("reset_device", True))
if parameters["method"] == "grub-efi" and reset_device:
# added unless the device specifies not to reset the device in grub.
self.internal_pipeline.add_action(ResetDevice())
elif parameters["method"] == "grub":
self.internal_pipeline.add_action(ResetDevice())
if parameters["method"] == "grub-efi":
self.internal_pipeline.add_action(UEFIMenuInterrupt())
self.internal_pipeline.add_action(GrubMenuSelector())
self.internal_pipeline.add_action(BootloaderInterruptAction())
self.internal_pipeline.add_action(BootloaderCommandsAction())
if self.has_prompts(parameters):
self.internal_pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.internal_pipeline.add_action(ExpectShellSession())
if "transfer_overlay" in parameters:
self.internal_pipeline.add_action(OverlayUnpack())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
else:
if self.has_boot_finished(parameters):
self.logger.debug("Doing a boot without a shell (installer)")
self.internal_pipeline.add_action(InstallerWait())
self.internal_pipeline.add_action(PowerOff())
def run(self, connection, max_end_time):
connection = super().run(connection, max_end_time)
self.set_namespace_data(action="shared",
label="shared",
key="connection",
value=connection)
return connection
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.internal_pipeline.add_action(BootPyOCDRetry())
class DepthchargeRetry(BootAction):
name = "depthcharge-retry"
description = "interactive depthcharge retry action"
summary = "depthcharge commands with retry"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.internal_pipeline.add_action(ResetDevice())
self.internal_pipeline.add_action(DepthchargeStart())
self.internal_pipeline.add_action(BootloaderCommandsAction())
if self.has_prompts(parameters):
self.internal_pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.internal_pipeline.add_action(ExpectShellSession())
if 'transfer_overlay' in parameters:
self.internal_pipeline.add_action(OverlayUnpack())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
def run(self, connection, max_end_time, args=None):
connection = super(DepthchargeRetry, self).run(connection,
max_end_time, args)
self.set_namespace_data(action='shared',
label='shared',
key='connection',
value=connection)
return connection
class FastbootFlashOrderAction(DeployAction):
"""
Fastboot flash image.
"""
name = "fastboot-flash-order-action"
description = "Determine support for each flash operation"
summary = "Handle reset and options for each flash url."
def __init__(self):
super().__init__()
self.retries = 3
self.sleep = 10
self.interrupt_prompt = None
self.interrupt_string = None
self.reboot = None
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
flash_cmds_order = self.job.device['flash_cmds_order']
userlist = list(parameters['images'].keys())
userlist.remove('yaml_line')
flash_cmds = set(userlist).difference(set(flash_cmds_order))
flash_cmds = flash_cmds_order + list(flash_cmds)
self.internal_pipeline.add_action(ReadFeedback(repeat=True))
for flash_cmd in flash_cmds:
if flash_cmd not in parameters['images']:
continue
self.internal_pipeline.add_action(
FastbootFlashAction(cmd=flash_cmd))
self.reboot = parameters['images'][flash_cmd].get('reboot')
if self.reboot == 'fastboot-reboot':
self.internal_pipeline.add_action(FastbootReboot())
self.internal_pipeline.add_action(ReadFeedback(repeat=True))
elif self.reboot == 'fastboot-reboot-bootloader':
self.internal_pipeline.add_action(FastbootRebootBootloader())
self.internal_pipeline.add_action(ReadFeedback(repeat=True))
elif self.reboot == 'hard-reset':
self.internal_pipeline.add_action(PDUReboot())
self.internal_pipeline.add_action(ReadFeedback(repeat=True))
def validate(self):
super().validate()
self.set_namespace_data(action=FastbootFlashAction.name,
label='interrupt',
key='reboot',
value=self.reboot)
if 'fastboot_serial_number' not in self.job.device:
self.errors = "device fastboot serial number missing"
elif self.job.device['fastboot_serial_number'] == '0000000000':
self.errors = "device fastboot serial number unset"
if 'flash_cmds_order' not in self.job.device:
self.errors = "device flash commands order missing"
if 'fastboot_options' not in self.job.device:
self.errors = "device fastboot options missing"
elif not isinstance(self.job.device['fastboot_options'], list):
self.errors = "device fastboot options is not a list"
class DepthchargeRetry(BootHasMixin, RetryAction):
name = "depthcharge-retry"
description = "interactive depthcharge retry action"
summary = "depthcharge commands with retry"
def populate(self, parameters):
self.pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.pipeline.add_action(ResetDevice())
self.pipeline.add_action(DepthchargeStart())
self.pipeline.add_action(BootloaderCommandsAction())
if self.has_prompts(parameters):
self.pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.pipeline.add_action(ExpectShellSession())
if "transfer_overlay" in parameters:
self.pipeline.add_action(OverlayUnpack())
self.pipeline.add_action(ExportDeviceEnvironment())
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.internal_pipeline.add_action(Unmount())
class VirtAction(DeployAction): # pylint:disable=too-many-instance-attributes
name = "virt-deploy"
description = "download images and deploy using virttool"
summary = "virt deployment"
def __init__(self):
super().__init__()
self.suffix = None
self.image_path = None
def validate(self):
super().validate()
# No need to go further if an error was already detected
if 'kernel' in self.parameters:
if 'rootfs' not in self.parameters:
self.errors = "kernel image exist, No rootfs image"
elif 'master' not in self.parameters:
self.errors = "master/kernel images all not exist"
if not self.valid:
return
# Extract the 3 last path elements. See action.mkdtemp()
suffix = os.path.join(*self.image_path.split('/')[-2:])
self.set_namespace_data(action=self.name,
label='storage',
key='suffix',
value=suffix)
if 'rootfs' in self.parameters:
suffix = os.path.join(*self.image_path.split('/')[-2:])
suffix = os.path.join(suffix, "rootfs")
self.set_namespace_data(action=self.name,
label='storage',
key='suffix',
value=suffix)
def populate(self, parameters):
"""
The dispatcher does the first download as the first deployment is not guaranteed to
have DNS resolution fully working, so we can use the IP address of the dispatcher
to get it (with the advantage that the dispatcher decompresses it so that the ramdisk
can pipe the raw image directly from wget to deploy with tools (dd,tar and so on).)
This also allows the use of local file:// locations which are visible to the dispatcher
but not the device.
"""
self.image_path = self.mkdtemp()
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
uniquify = parameters.get('uniquify', True)
self.internal_pipeline.add_action(VirtDeployMasterAction())
if 'rootfs_path' in parameters:
self.internal_pipeline.add_action(
DownloaderAction('rootfs',
path=self.image_path,
uniquify=uniquify))
self.internal_pipeline.add_action(VirtDeploySlaveAction())
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(
OverlayAction()) # idempotent, includes testdef
# FIXME: could support tarballs too
if self.test_needs_deployment(parameters):
self.internal_pipeline.add_action(DeployDeviceEnvironment())
def populate(self, parameters):
"""
Each time a test definition is processed by a handler, a new set of
overlay files are needed, based on that test definition. Basic overlay
files are created by TestOverlayAction. More complex scripts like the
install:deps script and the main run script have custom Actions.
"""
index = []
self.pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.test_list = identify_test_definitions(self.job.test_info,
parameters["namespace"])
if self.test_list:
self.set_namespace_data(
action=self.name,
label=self.name,
key="test_list",
value=self.test_list,
parameters=parameters,
)
for testdefs in self.test_list:
for testdef in testdefs:
# namespace support allows only running the install steps for the relevant
# deployment as the next deployment could be a different OS.
handler = RepoAction.select(testdef["from"])()
# set the full set of job YAML parameters for this handler as handler parameters.
handler.job = self.job
handler.parameters = testdef
# store the correct test_name before appending to the local index
handler.parameters["test_name"] = "%s_%s" % (
len(index),
handler.parameters["name"],
)
self.pipeline.add_action(handler)
# a genuinely unique ID based on the *database* JobID and
# pipeline level for reproducibility and tracking -
# {DB-JobID}_{PipelineLevel}, e.g. 15432.0_3.5.4
handler.uuid = "%s_%s" % (self.job.job_id, handler.level)
handler.stage = self.stages
self.run_levels[testdef["name"]] = self.stages
# copy details into the overlay, one per handler but the same class each time.
overlay = TestOverlayAction()
overlay.job = self.job
overlay.parameters = testdef
overlay.parameters["test_name"] = handler.parameters[
"test_name"]
overlay.test_uuid = handler.uuid
# add install handler - uses job parameters
installer = TestInstallAction()
installer.job = self.job
installer.parameters = testdef
installer.parameters["test_name"] = handler.parameters[
"test_name"]
installer.test_uuid = handler.uuid
# add runsh handler - uses job parameters
runsh = TestRunnerAction()
runsh.job = self.job
runsh.parameters = testdef
runsh.parameters["test_name"] = handler.parameters["test_name"]
runsh.test_uuid = handler.uuid
index.append(handler.parameters["name"])
# add overlay handlers to the pipeline
self.pipeline.add_action(overlay)
self.pipeline.add_action(installer)
self.pipeline.add_action(runsh)
self.set_namespace_data(
action="test-definition",
label="test-definition",
key="testdef_index",
value=index,
parameters=parameters,
)
self.stages += 1
class GrubSequenceAction(BootAction):
name = "grub-sequence-action"
description = "grub boot sequence"
summary = "run grub boot using specified sequence of actions"
def __init__(self):
super().__init__()
self.expect_shell = False
def validate(self):
super().validate()
sequences = self.job.device["actions"]["boot"]["methods"]["grub"].get(
"sequence", [])
for sequence in sequences:
if not _grub_sequence_map(sequence):
self.errors = "Unknown boot sequence '%s'" % sequence
def populate(self, parameters):
super().populate(parameters)
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
sequences = self.job.device["actions"]["boot"]["methods"]["grub"].get(
"sequence", [])
for sequence in sequences:
mapped = _grub_sequence_map(sequence)
if mapped[1]:
self.internal_pipeline.add_action(mapped[0](itype=mapped[1]))
elif mapped[0]:
self.internal_pipeline.add_action(mapped[0]())
if self.has_prompts(parameters):
self.internal_pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.internal_pipeline.add_action(ExpectShellSession())
if "transfer_overlay" in parameters:
self.internal_pipeline.add_action(OverlayUnpack())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
else:
if self.has_boot_finished(parameters):
self.logger.debug("Doing a boot without a shell (installer)")
self.internal_pipeline.add_action(InstallerWait())
self.internal_pipeline.add_action(PowerOff())
def test_overlay_action(self): # pylint: disable=too-many-locals
parameters = {
'device_type': 'beaglebone-black',
'job_name': 'uboot-pipeline',
'job_timeout': '15m',
'action_timeout': '5m',
'priority': 'medium',
'actions': {
'boot': {
'method': 'u-boot',
'commands': 'ramdisk',
'type': 'bootz',
'prompts': ['linaro-test', 'root@debian:~#']
},
'deploy': {
'ramdisk': 'initrd.gz',
'kernel': 'zImage',
'dtb': 'broken.dtb'
}
}
}
device = NewDevice(
os.path.join(os.path.dirname(__file__), '../devices/bbb-01.yaml'))
job = Job(4212, parameters, None)
job.device = device
pipeline = Pipeline(job=job, parameters=parameters['actions']['boot'])
job.pipeline = pipeline
overlay = BootloaderCommandOverlay()
pipeline.add_action(overlay)
ip_addr = dispatcher_ip(None)
parsed = []
kernel_addr = job.device['parameters'][
overlay.parameters['type']]['ramdisk']
ramdisk_addr = job.device['parameters'][
overlay.parameters['type']]['ramdisk']
dtb_addr = job.device['parameters'][overlay.parameters['type']]['dtb']
kernel = parameters['actions']['deploy']['kernel']
ramdisk = parameters['actions']['deploy']['ramdisk']
dtb = parameters['actions']['deploy']['dtb']
substitution_dictionary = {
'{SERVER_IP}':
ip_addr,
# the addresses need to be hexadecimal
'{KERNEL_ADDR}':
kernel_addr,
'{DTB_ADDR}':
dtb_addr,
'{RAMDISK_ADDR}':
ramdisk_addr,
'{BOOTX}':
"%s %s %s %s" %
(overlay.parameters['type'], kernel_addr, ramdisk_addr, dtb_addr),
'{RAMDISK}':
ramdisk,
'{KERNEL}':
kernel,
'{DTB}':
dtb
}
params = device['actions']['boot']['methods']
params['u-boot']['ramdisk']['commands'] = substitute(
params['u-boot']['ramdisk']['commands'], substitution_dictionary)
commands = params['u-boot']['ramdisk']['commands']
self.assertIs(type(commands), list)
self.assertIn("setenv loadkernel 'tftp ${kernel_addr_r} zImage'",
commands)
self.assertIn(
"setenv loadinitrd 'tftp ${initrd_addr_r} initrd.gz; setenv initrd_size ${filesize}'",
commands)
self.assertIn("setenv loadfdt 'tftp ${fdt_addr_r} broken.dtb'",
commands)
self.assertNotIn("setenv kernel_addr_r '{KERNEL_ADDR}'", commands)
self.assertNotIn("setenv initrd_addr_r '{RAMDISK_ADDR}'", commands)
self.assertNotIn("setenv fdt_addr_r '{DTB_ADDR}'", commands)
for line in params['u-boot']['ramdisk']['commands']:
line = line.replace('{SERVER_IP}', ip_addr)
# the addresses need to be hexadecimal
line = line.replace('{KERNEL_ADDR}', kernel_addr)
line = line.replace('{DTB_ADDR}', dtb_addr)
line = line.replace('{RAMDISK_ADDR}', ramdisk_addr)
line = line.replace(
'{BOOTX}',
"%s %s %s %s" % (overlay.parameters['type'], kernel_addr,
ramdisk_addr, dtb_addr))
line = line.replace('{RAMDISK}', ramdisk)
line = line.replace('{KERNEL}', kernel)
line = line.replace('{DTB}', dtb)
parsed.append(line)
self.assertIn("setenv loadkernel 'tftp ${kernel_addr_r} zImage'",
parsed)
self.assertIn(
"setenv loadinitrd 'tftp ${initrd_addr_r} initrd.gz; setenv initrd_size ${filesize}'",
parsed)
self.assertIn("setenv loadfdt 'tftp ${fdt_addr_r} broken.dtb'", parsed)
self.assertNotIn("setenv kernel_addr_r '{KERNEL_ADDR}'", parsed)
self.assertNotIn("setenv initrd_addr_r '{RAMDISK_ADDR}'", parsed)
self.assertNotIn("setenv fdt_addr_r '{DTB_ADDR}'", parsed)
class MinimalBoot(BootHasMixin, RetryAction):
name = "minimal-boot"
description = "connect and reset device"
summary = "connect and reset device"
def populate(self, parameters):
self.pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.pipeline.add_action(ConnectDevice())
if parameters.get("pre_power_command", False):
self.pipeline.add_action(PrePower())
if parameters.get("pre_os_command", False):
self.pipeline.add_action(PreOs())
if parameters.get("reset", True):
self.pipeline.add_action(ResetDevice())
if self.has_prompts(parameters):
self.pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.pipeline.add_action(ExpectShellSession())
if "transfer_overlay" in parameters:
self.pipeline.add_action(OverlayUnpack())
self.pipeline.add_action(ExportDeviceEnvironment())
class PrepareOverlayTftp(Action):
"""
Extracts the ramdisk or nfsrootfs in preparation for the lava overlay
"""
name = "prepare-tftp-overlay"
description = "extract ramdisk or nfsrootfs in preparation for lava overlay"
summary = "extract ramdisk or nfsrootfs"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.internal_pipeline.add_action(
ExtractNfsRootfs()) # idempotent, checks for nfsrootfs parameter
self.internal_pipeline.add_action(
OverlayAction()) # idempotent, includes testdef
self.internal_pipeline.add_action(
ExtractRamdisk()) # idempotent, checks for a ramdisk parameter
self.internal_pipeline.add_action(
ExtractModules()) # idempotent, checks for a modules parameter
self.internal_pipeline.add_action(ApplyOverlayTftp())
if 'kernel' in parameters and 'type' in parameters['kernel']:
self.internal_pipeline.add_action(PrepareKernelAction())
self.internal_pipeline.add_action(ConfigurePreseedFile(
)) # idempotent, checks for a preseed parameter
self.internal_pipeline.add_action(
CompressRamdisk()) # idempotent, checks for a ramdisk parameter
if 'depthcharge' in self.job.device['actions']['boot']['methods']:
self.internal_pipeline.add_action(PrepareKernelAction())
def run(self, connection, max_end_time, args=None):
connection = super(PrepareOverlayTftp,
self).run(connection, max_end_time, args)
ramdisk = self.get_namespace_data(action='download-action',
label='file',
key='ramdisk')
if ramdisk: # nothing else to do
return connection
return connection
class MpsAction(Action):
"""
Action for deploying firmware to a MPS board in the form
of a board recovery image. Recovery images must have AUTORUN
set to true in config.txt in order for the device to come to
a prompt after reboot.
"""
name = "mps-deploy"
description = "deploy image to MPS device"
summary = "MPS device image deployment"
def validate(self):
super().validate()
if "images" not in self.parameters:
self.errors = "Missing 'images'"
return
images = list(self.parameters["images"].keys())
if len(images) == 1:
if images[0] not in ["recovery_image", "test_binary"]:
self.errors = "Missing 'recovery_image' or 'test_binary'"
else:
for image in images:
if not image == "recovery_image" and not image.startswith(
"test_binary_"):
self.errors = (
"Missing 'recovery_image' or not starting with 'test_binary_'"
)
def populate(self, parameters):
download_dir = self.mkdtemp()
self.pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.pipeline.add_action(DisconnectDevice())
self.pipeline.add_action(ResetDevice())
self.pipeline.add_action(WaitUSBMassStorageDeviceAction())
for image in parameters["images"].keys():
self.pipeline.add_action(
DownloaderAction(image,
path=download_dir,
params=parameters["images"][image]))
self.pipeline.add_action(MountVExpressMassStorageDevice())
# Sort the keys so recovery_image will be first
for image in sorted(parameters["images"].keys()):
if image == "recovery_image":
self.pipeline.add_action(ExtractVExpressRecoveryImage())
self.pipeline.add_action(DeployVExpressRecoveryImage())
else:
self.pipeline.add_action(DeployMPSTestBinary(image))
# Should we hard reboot the board after flash?
params = self.job.device["actions"]["deploy"]["methods"]["mps"][
"parameters"]
if params["hard-reboot"]:
# Unmount the mass storage device before rebooting
self.pipeline.add_action(UnmountVExpressMassStorageDevice())
self.pipeline.add_action(PowerOff())
else:
# Unmount the mass storage device after the creation of reboot.txt
self.pipeline.add_action(DeployMPSRebootTxt())
self.pipeline.add_action(UnmountVExpressMassStorageDevice())
def test_keep_connection(self):
pipe = Pipeline()
pipe.add_action(TestFakeActions.KeepConnection())
conn = object()
self.assertIs(conn, pipe.run_actions(conn, None))
class TftpAction(Action):
name = "tftp-deploy"
description = "download files and deploy using tftp"
summary = "tftp deployment"
def __init__(self):
super().__init__()
self.tftp_dir = None
def validate(self):
super().validate()
if "kernel" not in self.parameters:
self.errors = "%s needs a kernel to deploy" % self.name
if not self.valid:
return
if "nfsrootfs" in self.parameters and "persistent_nfs" in self.parameters:
self.errors = "Only one of nfsrootfs or persistent_nfs can be specified"
which("in.tftpd")
# Check that the tmp directory is in the tftpd_dir or in /tmp for the
# unit tests
tftpd_directory = os.path.realpath(filesystem.tftpd_dir())
tftp_dir = os.path.realpath(self.tftp_dir)
tmp_dir = tempfile.gettempdir()
if not tftp_dir.startswith(
tftpd_directory) and not tftp_dir.startswith(tmp_dir):
self.errors = "tftpd directory is not configured correctly, see /etc/default/tftpd-hpa"
def populate(self, parameters):
self.tftp_dir = self.mkdtemp(override=filesystem.tftpd_dir())
self.pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.set_namespace_data(
action=self.name,
label="tftp",
key="tftp_dir",
value=self.tftp_dir,
parameters=parameters,
)
for key in [
"ramdisk", "kernel", "dtb", "nfsrootfs", "modules", "preseed"
]:
if key in parameters:
self.pipeline.add_action(
DownloaderAction(key,
path=self.tftp_dir,
params=parameters[key]))
if key == "ramdisk":
self.set_namespace_data(
action=self.name,
label="tftp",
key="ramdisk",
value=True,
parameters=parameters,
)
# TftpAction is a deployment, so once the files are in place, just do the overlay
self.pipeline.add_action(PrepareOverlayTftp())
self.pipeline.add_action(LxcCreateUdevRuleAction())
if self.test_needs_deployment(parameters):
self.pipeline.add_action(DeployDeviceEnvironment())
def run(self, connection, max_end_time):
# Extract the 3 last path elements. See action.mkdtemp()
suffix = os.path.join(*self.tftp_dir.split("/")[-2:])
self.set_namespace_data(action=self.name,
label="tftp",
key="suffix",
value=suffix)
super().run(connection, max_end_time)
tftp_size_limit = self.job.parameters["dispatcher"].get(
"tftp_size_limit", TFTP_SIZE_LIMIT)
self.logger.debug("Checking files for TFTP limit of %s bytes.",
tftp_size_limit)
for (action, key) in [
("compress-ramdisk", "ramdisk"),
("download-action", "kernel"),
("download-action", "dtb"),
]:
if key in self.parameters:
filename = self.get_namespace_data(action=action,
label="file",
key=key)
filename = os.path.join(filesystem.tftpd_dir(), filename)
fsize = os.stat(filename).st_size
if fsize >= tftp_size_limit:
raise JobError(
"Unable to send '%s' over tftp: file too large (%d > %d)"
% (os.path.basename(filename), fsize, tftp_size_limit))
return connection
def test_complex_pipeline(self):
action = Action()
action.name = "starter_action"
action.description = "test action only"
action.summary = "starter"
pipe = Pipeline()
pipe.add_action(action)
self.assertEqual(action.level, "1")
action = Action()
action.name = "pipe_action"
action.description = "action implementing an internal pipe"
action.summary = "child"
pipe.add_action(action)
self.assertEqual(action.level, "2")
# a formal RetryAction would contain a pre-built pipeline which can be inserted directly
retry_pipe = Pipeline(action)
action = Action()
action.name = "child_action"
action.description = "action inside the internal pipe"
action.summary = "child"
retry_pipe.add_action(action)
self.assertEqual(action.level, "2.1")
action = Action()
action.name = "second-child-action"
action.description = "second action inside the internal pipe"
action.summary = "child2"
retry_pipe.add_action(action)
self.assertEqual(action.level, "2.2")
action = Action()
action.name = "baby_action"
action.description = "action implementing an internal pipe"
action.summary = "baby"
retry_pipe.add_action(action)
self.assertEqual(action.level, "2.3")
inner_pipe = Pipeline(action)
action = Action()
action.name = "single_action"
action.description = "single line action"
action.summary = "single"
inner_pipe.add_action(action)
self.assertEqual(action.level, "2.3.1")
action = Action()
action.name = "step_out"
action.description = "step out of inner pipe"
action.summary = "brother"
retry_pipe.add_action(action)
self.assertEqual(action.level, "2.4")
action = Action()
action.name = "top-level"
action.description = "top level"
action.summary = "action"
pipe.add_action(action)
self.assertEqual(action.level, "3")
self.assertEqual(len(pipe.describe()), 3)
class MinimalBoot(BootAction):
name = 'minimal-boot'
description = "connect and reset device"
summary = "connect and reset device"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.internal_pipeline.add_action(ResetDevice())
self.internal_pipeline.add_action(ConnectDevice())
if self.has_prompts(parameters):
self.internal_pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.internal_pipeline.add_action(ExpectShellSession())
if 'transfer_overlay' in parameters:
self.internal_pipeline.add_action(OverlayUnpack())
self.internal_pipeline.add_action(ExportDeviceEnvironment())
def run(self, connection, max_end_time):
connection = super().run(connection, max_end_time)
self.set_namespace_data(action='shared',
label='shared',
key='connection',
value=connection)
return connection
def test_create_empty_pipeline(self):
pipe = Pipeline()
self.assertEqual(pipe.actions, [])
class UrlRepoAction(RepoAction):
priority = 1
name = "url-repo-action"
description = "apply a single test file to the test image"
summary = "download file test"
def __init__(self):
super().__init__()
self.tmpdir = None # FIXME: needs to be a /mntpoint/lava-%hostname/ directory.
self.testdef = None
@classmethod
def accepts(cls, repo_type):
return repo_type == "url"
def validate(self):
if "repository" not in self.parameters:
self.errors = "Url repository not specified in job definition"
if "path" not in self.parameters:
self.errors = "Path to YAML file not specified in the job definition"
super().validate()
def populate(self, parameters):
# Import the module here to avoid cyclic import.
from lava_dispatcher.actions.deploy.download import DownloaderAction
# Add 'url' as an alias to 'repository'. DownloaderAction requires an
# 'url' key.
params = dict(**parameters, url=self.parameters["repository"])
self.download_dir = self.mkdtemp()
self.action_key = "url_repo"
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=params)
self.internal_pipeline.add_action(
DownloaderAction(self.action_key, self.download_dir,
params=params))
def run(self, connection, max_end_time):
"""Download the provided test definition file into tmpdir."""
super().run(connection, max_end_time)
runner_path = self.get_namespace_data(action="uuid",
label="overlay_path",
key=self.parameters["test_name"])
fname = self.get_namespace_data(action="download-action",
label=self.action_key,
key="file")
self.logger.debug("Runner path : %s", runner_path)
if os.path.exists(runner_path) and os.listdir(runner_path) == []:
raise LAVABug(
"Directory already exists and is not empty - duplicate Action?"
)
self.logger.info("Untar tests from file %s to directory %s", fname,
runner_path)
untar_file(fname, runner_path)
# now read the YAML to create a testdef dict to retrieve metadata
yaml_file = os.path.join(runner_path, self.parameters["path"])
self.logger.debug("Tests stored (tmp) in %s", yaml_file)
try:
with open(yaml_file, "r") as test_file:
testdef = yaml_safe_load(test_file)
except IOError as exc:
raise JobError("Unable to open test definition '%s': %s" %
(self.parameters["path"], str(exc)))
# set testdef metadata in base class
self.store_testdef(testdef, "url")
return connection
class FastbootAction(DeployAction): # pylint:disable=too-many-instance-attributes
name = "fastboot-deploy"
description = "download files and deploy using fastboot"
summary = "fastboot deployment"
def __init__(self):
super().__init__()
self.force_prompt = False
def validate(self):
super().validate()
if not self.test_needs_deployment(self.parameters):
return
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(OverlayAction())
# Check if the device has a power command such as HiKey, Dragonboard,
# etc. against device that doesn't like Nexus, etc.
if self.job.device.get('fastboot_via_uboot', False):
self.internal_pipeline.add_action(ConnectDevice())
self.internal_pipeline.add_action(UBootEnterFastbootAction())
elif self.job.device.hard_reset_command:
self.force_prompt = True
self.internal_pipeline.add_action(ConnectDevice())
if not is_lxc_requested(self.job):
self.internal_pipeline.add_action(PrePower())
self.internal_pipeline.add_action(ResetDevice())
else:
self.internal_pipeline.add_action(EnterFastbootAction())
fastboot_dir = self.mkdtemp()
image_keys = sorted(parameters['images'].keys())
for image in image_keys:
if image != 'yaml_line':
self.internal_pipeline.add_action(
DownloaderAction(image, fastboot_dir))
if parameters['images'][image].get('apply-overlay', False):
if self.test_needs_overlay(parameters):
if parameters['images'][image].get('sparse', True):
self.internal_pipeline.add_action(
ApplyOverlaySparseImage(image))
else:
self.internal_pipeline.add_action(
ApplyOverlayImage(image,
use_root_partition=False))
if self.test_needs_overlay(parameters) and \
self.test_needs_deployment(parameters):
self.internal_pipeline.add_action(
DeployDeviceEnvironment())
self.internal_pipeline.add_action(FastbootFlashOrderAction())
class NfsAction(Action):
name = "nfs-deploy"
description = "deploy nfsrootfs"
summary = "NFS deployment"
def validate(self):
super().validate()
if not self.valid:
return
if "nfsrootfs" in self.parameters and "persistent_nfs" in self.parameters:
self.errors = "Only one of nfsrootfs or persistent_nfs can be specified"
def populate(self, parameters):
download_dir = self.mkdtemp()
self.pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
if "nfsrootfs" in parameters:
self.pipeline.add_action(
DownloaderAction(
"nfsrootfs", path=download_dir, params=parameters["nfsrootfs"]
)
)
if "modules" in parameters:
self.pipeline.add_action(
DownloaderAction(
"modules", path=download_dir, params=parameters["modules"]
)
)
# NfsAction is a deployment, so once the nfsrootfs has been deployed, just do the overlay
self.pipeline.add_action(ExtractNfsRootfs())
self.pipeline.add_action(OverlayAction())
self.pipeline.add_action(ExtractModules())
self.pipeline.add_action(ApplyOverlayTftp())
if self.test_needs_deployment(parameters):
self.pipeline.add_action(DeployDeviceEnvironment())
class DeployQemuNfsAction(DeployAction):
name = "deploy-qemu-nfs"
description = "deploy qemu with NFS"
summary = "deploy NFS for QEMU"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
path = self.mkdtemp()
if "uefi" in parameters:
uefi_path = self.mkdtemp()
self.internal_pipeline.add_action(
DownloaderAction("uefi", uefi_path))
# uefi option of QEMU needs a directory, not the filename
self.set_namespace_data(
action=self.name,
label="image",
key="uefi_dir",
value=uefi_path,
parameters=parameters,
)
# alternatively use the -bios option and standard image args
for image in parameters["images"].keys():
self.internal_pipeline.add_action(DownloaderAction(image, path))
if parameters["images"][image].get("format", "") == "qcow2":
self.internal_pipeline.add_action(QCowConversionAction(image))
self.internal_pipeline.add_action(ExtractNfsAction())
self.internal_pipeline.add_action(OverlayAction())
self.internal_pipeline.add_action(ApplyOverlayTftp())
self.internal_pipeline.add_action(DeployDeviceEnvironment())
class DeployImagesAction(DeployAction): # FIXME: Rename to DeployPosixImages
name = "deployimages"
description = "deploy images using guestfs"
summary = "deploy images"
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
path = self.mkdtemp()
if "uefi" in parameters:
uefi_path = self.mkdtemp()
self.internal_pipeline.add_action(
DownloaderAction("uefi", uefi_path))
# uefi option of QEMU needs a directory, not the filename
self.set_namespace_data(
action=self.name,
label="image",
key="uefi_dir",
value=uefi_path,
parameters=parameters,
)
# alternatively use the -bios option and standard image args
for image in parameters["images"].keys():
self.internal_pipeline.add_action(DownloaderAction(image, path))
if parameters["images"][image].get("format", "") == "qcow2":
self.internal_pipeline.add_action(QCowConversionAction(image))
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(
OverlayAction()) # idempotent, includes testdef
self.internal_pipeline.add_action(ApplyOverlayGuest())
if self.test_needs_deployment(parameters):
self.internal_pipeline.add_action(DeployDeviceEnvironment())
class LxcAction(DeployAction): # pylint:disable=too-many-instance-attributes
name = "lxc-deploy"
description = "download files and deploy using lxc"
summary = "lxc deployment"
def __init__(self):
super().__init__()
self.lxc_data = {}
def validate(self):
super().validate()
lxc_version = debian_package_version(pkg='lxc', split=False)
if lxc_version is not '':
self.logger.info("lxc, installed at version: %s", lxc_version)
else:
self.logger.info(
"lava-lxc-mocker, installed at version: %s",
debian_package_version(pkg='lava-lxc-mocker', split=False))
protocols = [protocol.name for protocol in self.job.protocols]
if LxcProtocol.name not in protocols:
self.logger.debug("Missing protocol '%s' in %s", LxcProtocol.name,
protocols)
self.errors = "Missing protocol '%s'" % LxcProtocol.name
which('lxc-create')
def populate(self, parameters):
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.internal_pipeline.add_action(LxcCreateAction())
self.internal_pipeline.add_action(LxcCreateUdevRuleAction())
if 'packages' in parameters:
self.internal_pipeline.add_action(LxcStartAction())
self.internal_pipeline.add_action(LxcAptUpdateAction())
self.internal_pipeline.add_action(LxcAptInstallAction())
self.internal_pipeline.add_action(LxcStopAction())
if self.test_needs_deployment(parameters):
self.internal_pipeline.add_action(DeployDeviceEnvironment())
if self.test_needs_overlay(parameters):
self.internal_pipeline.add_action(OverlayAction())
self.internal_pipeline.add_action(ApplyLxcOverlay())
class BootFastbootAction(BootAction):
"""
Provide for auto_login parameters in this boot stanza and re-establish the
connection after boot.
"""
name = "fastboot-boot"
description = "fastboot boot into the system"
summary = "fastboot boot"
def validate(self):
super().validate()
sequences = self.job.device["actions"]["boot"]["methods"].get("fastboot", [])
if sequences is not None:
for sequence in sequences:
if not _fastboot_sequence_map(sequence):
self.errors = "Unknown boot sequence '%s'" % sequence
else:
self.errors = "fastboot_sequence undefined"
def populate(self, parameters):
self.pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
if parameters.get("commands"):
self.pipeline.add_action(BootFastbootCommands())
board_id = self.job.device["fastboot_serial_number"]
# Always ensure the device is in fastboot mode before trying to boot.
# Check if the device has a power command such as HiKey, Dragonboard,
# etc. against device that doesn't like Nexus, etc.
if self.job.device.get("fastboot_via_uboot", False):
self.pipeline.add_action(ConnectDevice())
self.pipeline.add_action(UBootEnterFastbootAction())
elif self.job.device.hard_reset_command:
self.force_prompt = True
self.pipeline.add_action(ConnectDevice())
self.pipeline.add_action(ResetDevice())
else:
self.pipeline.add_action(WaitDeviceBoardID(board_id))
self.pipeline.add_action(EnterFastbootAction())
# Based on the boot sequence defined in the device configuration, add
# the required pipeline actions.
sequences = self.job.device["actions"]["boot"]["methods"].get("fastboot", [])
for sequence in sequences:
mapped = _fastboot_sequence_map(sequence)
self.pipeline.add_action(WaitDeviceBoardID(board_id))
if mapped[1]:
self.pipeline.add_action(mapped[0](device_actions=mapped[1]))
elif mapped[0]:
self.pipeline.add_action(mapped[0]())
if self.job.device.hard_reset_command:
if not is_lxc_requested(self.job):
self.pipeline.add_action(PreOs())
if self.has_prompts(parameters):
self.pipeline.add_action(AutoLoginAction())
if self.test_has_shell(parameters):
self.pipeline.add_action(ExpectShellSession())
if "transfer_overlay" in parameters:
self.pipeline.add_action(OverlayUnpack())
self.pipeline.add_action(ExportDeviceEnvironment())
else:
if not is_lxc_requested(self.job):
self.pipeline.add_action(ConnectAdb())
self.pipeline.add_action(AdbOverlayUnpack())
