def run(self, connection, max_end_time, args=None):
output = os.path.join(self.job.parameters['output_dir'],
"overlay-%s.tar.gz" % self.level)
location = self.get_namespace_data(action='test', label='shared', key='location')
lava_test_results_dir = self.get_namespace_data(action='test', label='results', key='lava_test_results_dir')
self.set_namespace_data(action='test', label='shared', key='output', value=output)
if not location:
raise LAVABug("Missing lava overlay location")
if not os.path.exists(location):
raise LAVABug("Unable to find overlay location")
if not self.valid:
self.logger.error(self.errors)
return connection
connection = super(CompressOverlay, self).run(connection, max_end_time, args)
cur_dir = os.getcwd()
try:
with tarfile.open(output, "w:gz") as tar:
os.chdir(location)
tar.add(".%s" % lava_test_results_dir)
# ssh authorization support
if os.path.exists('./root/'):
tar.add(".%s" % '/root/')
except tarfile.TarError as exc:
raise InfrastructureError("Unable to create lava overlay tarball: %s" % exc)
os.chdir(cur_dir)
self.set_namespace_data(action=self.name, label='output', key='file', value=output)
return connection
def wait_udev_event(action='add', match_dict=None, subsystem=None, devtype=None, devicepath=None):
if action not in ['add', 'remove', 'change']:
raise LAVABug("Invalid action for udev to wait for: %s, expected 'add' or 'remove'" % action)
if match_dict:
if not isinstance(match_dict, dict):
raise LAVABug("match_dict was not a dict")
else:
if devicepath:
if not isinstance(devicepath, str):
raise LAVABug("devicepath was not a string")
match_dict = {}
else:
raise LAVABug("Neither match_dict nor devicepath were set")
if devtype and not subsystem:
raise LAVABug("Cannot filter udev by devtype without a subsystem")
match_dict['ACTION'] = action
context = pyudev.Context()
monitor = pyudev.Monitor.from_netlink(context)
if devtype and subsystem:
monitor.filter_by(subsystem, devtype)
else:
if subsystem:
monitor.filter_by(subsystem)
for device in iter(monitor.poll, None):
same = _dict_compare(dict(device), match_dict)
if same == set(match_dict.keys()):
if devicepath:
if devicepath in dict(device).get('DEVLINKS', '') or \
devicepath in dict(device).get('DEVNAME', ''):
break
else:
break
def run(self, connection, max_end_time, args=None):
connection = super(SshAuthorize, self).run(connection, max_end_time, args)
if not self.identity_file:
self.logger.debug("No authorisation required.") # idempotency
return connection
# add the authorization keys to the overlay
location = self.get_namespace_data(action='test', label='shared', key='location')
lava_test_results_dir = self.get_namespace_data(action='test', label='results', key='lava_test_results_dir')
if not location:
raise LAVABug("Missing lava overlay location")
if not os.path.exists(location):
raise LAVABug("Unable to find overlay location")
lava_path = os.path.abspath("%s/%s" % (location, lava_test_results_dir))
output_file = '%s/%s' % (lava_path, os.path.basename(self.identity_file))
shutil.copyfile(self.identity_file, output_file)
shutil.copyfile("%s.pub" % self.identity_file, "%s.pub" % output_file)
if not self.active:
# secondary connections only
return connection
self.logger.info("Adding SSH authorisation for %s.pub", os.path.basename(output_file))
user_sshdir = os.path.join(location, 'root', '.ssh')
if not os.path.exists(user_sshdir):
os.makedirs(user_sshdir, 0o755)
# if /root/.ssh/authorized_keys exists in the test image it will be overwritten
# the key exists in the lava_test_results_dir to allow test writers to work around this
# after logging in via the identity_file set here
authorize = os.path.join(user_sshdir, 'authorized_keys')
self.logger.debug("Copying %s to %s", "%s.pub" % self.identity_file, authorize)
shutil.copyfile("%s.pub" % self.identity_file, authorize)
os.chmod(authorize, 0o600)
return connection
def wait_udev_event(action='add',
match_dict=None,
subsystem=None,
devtype=None):
if action not in ['add', 'remove']:
raise LAVABug(
"Invalid action for udev to wait for: %s, expected 'add' or 'remove'"
% action)
if match_dict:
if isinstance(match_dict, dict):
if match_dict == {}:
raise LAVABug(
"Trying to match udev event with empty match_dict")
else:
raise LAVABug("match_dict was not a dict")
else:
raise LAVABug("match_dict was None")
if devtype and not subsystem:
raise LAVABug("Cant filter udev by devtype without a subsystem")
match_dict['ACTION'] = action
context = pyudev.Context()
monitor = pyudev.Monitor.from_netlink(context)
if devtype and subsystem:
monitor.filter_by(subsystem, devtype)
else:
if subsystem:
monitor.filter_by(subsystem)
for device in iter(monitor.poll, None):
same = _dict_compare(dict(device), match_dict)
if same == set(match_dict.keys()):
break
def run(self, connection, max_end_time, args=None):
"""
Writes out file contents from lists, across multiple lines
VAR="VAL1\n\
VAL2\n\
"
The \n and \ are used to avoid unwanted whitespace, so are escaped.
\n becomes \\n, \ becomes \\, which itself then needs \n to output:
VAL1
VAL2
"""
if not self.params:
self.logger.debug("skipped %s", self.name)
return connection
location = self.get_namespace_data(action='test', label='shared', key='location')
lava_test_results_dir = self.get_namespace_data(action='test', label='results', key='lava_test_results_dir')
shell = self.get_namespace_data(action='test', label='shared', key='lava_test_sh_cmd')
if not location:
raise LAVABug("Missing lava overlay location")
if not os.path.exists(location):
raise LAVABug("Unable to find overlay location")
lava_path = os.path.abspath("%s/%s" % (location, lava_test_results_dir))
scripts_to_copy = glob.glob(os.path.join(self.lava_vland_test_dir, 'lava-*'))
self.logger.debug(self.lava_vland_test_dir)
self.logger.debug({"lava_path": lava_path, "scripts": scripts_to_copy})
for fname in scripts_to_copy:
with open(fname, 'r') as fin:
foutname = os.path.basename(fname)
output_file = '%s/bin/%s' % (lava_path, foutname)
self.logger.debug("Creating %s", output_file)
with open(output_file, 'w') as fout:
fout.write("#!%s\n\n" % shell)
# Target-specific scripts (add ENV to the generic ones)
if foutname == 'lava-vland-self':
fout.write(r'LAVA_VLAND_SELF="')
for line in self.sysfs:
fout.write(r"%s\n" % line)
elif foutname == 'lava-vland-names':
fout.write(r'LAVA_VLAND_NAMES="')
for line in self.names:
fout.write(r"%s\n" % line)
elif foutname == 'lava-vland-tags':
fout.write(r'LAVA_VLAND_TAGS="')
if not self.tags:
fout.write(r"\n")
else:
for line in self.tags:
fout.write(r"%s\n" % line)
fout.write('"\n\n')
fout.write(fin.read())
os.fchmod(fout.fileno(), self.xmod)
self.call_protocols()
return connection
def _validate(self, simulate):
"""
Validate the pipeline and raise an exception (that inherit from
LAVAError) if it fails.
If simulate is True, then print the pipeline description.
"""
label = "lava-dispatcher, installed at version: %s" % debian_package_version(
split=False)
self.logger.info(label)
self.logger.info("start: 0 validate")
start = time.time()
for protocol in self.protocols:
try:
protocol.configure(self.device, self)
except KeyboardInterrupt:
self.logger.info("Canceled")
raise JobError("Canceled")
except LAVAError:
self.logger.error("Configuration failed for protocol %s",
protocol.name)
raise
except Exception as exc:
self.logger.error("Configuration failed for protocol %s",
protocol.name)
self.logger.exception(traceback.format_exc())
raise LAVABug(exc)
if not protocol.valid:
msg = "protocol %s has errors: %s" % (protocol.name,
protocol.errors)
self.logger.exception(msg)
raise JobError(msg)
if simulate:
# output the content and then any validation errors (python3 compatible)
print(yaml.dump(self.describe())) # pylint: disable=superfluous-parens
try:
self.pipeline.validate_actions()
except KeyboardInterrupt:
self.logger.info("Canceled")
raise JobError("Canceled")
except LAVAError as exc:
self.logger.error("Invalid job definition")
self.logger.exception(str(exc))
# This should be re-raised to end the job
raise
except Exception as exc:
self.logger.error("Validation failed")
self.logger.exception(traceback.format_exc())
raise LAVABug(exc)
finally:
self.logger.info("validate duration: %.02f", time.time() - start)
def _api_select(self, data, action=None):
if not data:
raise TestError("Protocol called without any data")
if not action:
raise LAVABug('LXC protocol needs to be called from an action.')
for item in data:
if 'request' not in item:
raise LAVABug("Malformed protocol request data.")
if 'pre-os-command' in item['request']:
action.logger.info("Running pre OS command via protocol.")
command = action.job.device.pre_os_command
if not action.run_command(command.split(' '), allow_silent=True):
raise InfrastructureError("%s failed" % command)
def run(self, connection, max_end_time, args=None):
if connection:
raise LAVABug(
"Fake action not meant to have a real connection")
time.sleep(3)
self.results = {'status': "failed"}
return connection
def __init__(self, command, lava_timeout, logger=None, cwd=None):
if not lava_timeout or not isinstance(lava_timeout, Timeout):
raise LAVABug("ShellCommand needs a timeout set by the calling Action")
if not logger:
raise LAVABug("ShellCommand needs a logger")
pexpect.spawn.__init__(
self, command,
timeout=lava_timeout.duration,
cwd=cwd,
logfile=ShellLogger(logger),
)
self.name = "ShellCommand"
self.logger = logger
# set a default newline character, but allow actions to override as neccessary
self.linesep = LINE_SEPARATOR
self.lava_timeout = lava_timeout
def run(self, connection, max_end_time, args=None):
if not connection:
raise LAVABug("%s started without a connection already in use" %
self.name)
connection = super(UBootInterrupt, self).run(connection, max_end_time,
args)
device_methods = self.job.device['actions']['boot']['methods']
# device is to be put into a reset state, either by issuing 'reboot' or power-cycle
interrupt_prompt = device_methods['u-boot']['parameters'].get(
'interrupt_prompt',
self.job.device.get_constant('uboot-autoboot-prompt'))
# interrupt_char can actually be a sequence of ASCII characters - sendline does not care.
interrupt_char = device_methods['u-boot']['parameters'].get(
'interrupt_char',
self.job.device.get_constant('uboot-interrupt-character'))
# vendor u-boot builds may require one or more control characters
interrupt_control_chars = device_methods['u-boot']['parameters'].get(
'interrupt_ctrl_list', [])
self.logger.debug("Changing prompt to '%s'", interrupt_prompt)
connection.prompt_str = interrupt_prompt
self.wait(connection)
if interrupt_control_chars:
for char in interrupt_control_chars:
connection.sendcontrol(char)
else:
connection.sendline(interrupt_char)
return connection
def _run(self):
"""
Run the pipeline under the run() wrapper that will catch the exceptions
"""
# Set the signal handler
signal.signal(signal.SIGINT, self.cancelling_handler)
signal.signal(signal.SIGTERM, self.cancelling_handler)
# Setup the protocols
for protocol in self.protocols:
try:
protocol.set_up()
except LAVAError:
raise
except Exception as exc:
self.logger.error("Unable to setup the protocols")
self.logger.exception(traceback.format_exc())
raise LAVABug(exc)
if not protocol.valid:
msg = "protocol %s has errors: %s" % (protocol.name,
protocol.errors)
self.logger.exception(msg)
raise JobError(msg)
# Run the pipeline and wait for exceptions
with self.timeout() as max_end_time:
self.pipeline.run_actions(self.connection, max_end_time)
def run(self, connection, max_end_time, args=None):
connection = super(ScpOverlayUnpack, self).run(connection,
max_end_time, args)
if not connection:
raise LAVABug("Cannot unpack, no connection available.")
filename = self.get_namespace_data(action='scp-deploy',
label='scp-overlay-unpack',
key='overlay')
tar_flags = self.get_namespace_data(action='scp-overlay',
label='scp-overlay',
key='tar_flags')
cmd = "tar %s -C / -xzf /%s" % (tar_flags, filename)
connection.sendline(cmd)
self.wait(connection)
self.set_namespace_data(action='shared',
label='shared',
key='connection',
value=connection)
res = 'failed' if self.errors else 'success'
self.set_namespace_data(action='boot',
label='shared',
key='boot-result',
value=res)
self.set_namespace_data(action='shared',
label='shared',
key='connection',
value=connection)
return connection
def run(self, connection, max_end_time, args=None):
if not self.parameters.get('ramdisk', None): # idempotency
return connection
ramdisk = self.get_namespace_data(action='download-action',
label='ramdisk',
key='file')
if self.skip:
self.logger.info("Not extracting ramdisk.")
suffix = self.get_namespace_data(action='tftp-deploy',
label='tftp',
key='suffix')
filename = os.path.join(suffix, "ramdisk",
os.path.basename(ramdisk))
# declare the original ramdisk as the name to be used later.
self.set_namespace_data(action='compress-ramdisk',
label='file',
key='ramdisk',
value=filename)
return
connection = super(ExtractRamdisk, self).run(connection, max_end_time,
args)
ramdisk_dir = self.mkdtemp()
extracted_ramdisk = os.path.join(ramdisk_dir, 'ramdisk')
os.mkdir(extracted_ramdisk, 0o755)
compression = self.parameters['ramdisk'].get('compression', None)
suffix = ""
if compression:
suffix = ".%s" % compression
ramdisk_compressed_data = os.path.join(ramdisk_dir,
RAMDISK_FNAME + suffix)
if self.parameters['ramdisk'].get('header', None) == 'u-boot':
cmd = ('dd if=%s of=%s ibs=%s skip=1' %
(ramdisk, ramdisk_compressed_data,
UBOOT_DEFAULT_HEADER_LENGTH)).split(' ')
try:
self.run_command(cmd)
except:
raise LAVABug('Unable to remove uboot header: %s' % ramdisk)
else:
# give the file a predictable name
shutil.move(ramdisk, ramdisk_compressed_data)
ramdisk_data = decompress_file(ramdisk_compressed_data, compression)
pwd = os.getcwd()
os.chdir(extracted_ramdisk)
cmd = ('cpio -iud -F %s' % ramdisk_data).split(' ')
if not self.run_command(cmd):
raise JobError(
'Unable to extract cpio archive: %s - missing header definition (i.e. u-boot)?'
% ramdisk_data)
os.chdir(pwd)
# tell other actions where the unpacked ramdisk can be found
self.set_namespace_data(action=self.name,
label='extracted_ramdisk',
key='directory',
value=extracted_ramdisk)
self.set_namespace_data(action=self.name,
label='ramdisk_file',
key='file',
value=ramdisk_data)
return connection
def run(self, connection, max_end_time, args=None):
if connection:
raise LAVABug(
"Fake action not meant to have a real connection")
connection = TestAdjuvant.FakeConnection()
self.results = {'status': "passed"}
self.data[TestAdjuvant.FakeAdjuvant.key()] = False
return connection
def run(self, connection, max_end_time, args=None):
connection = super(MenuReset, self).run(connection, max_end_time, args)
if not connection:
raise LAVABug("%s needs a Connection")
connection.check_char = '\n'
connection.sendline('\n') # to catch the first prompt (remove for PDU?)
return connection
def validate(self):
"""
The reasoning here is that the RetryAction should be in charge of an internal pipeline
so that the retry logic only occurs once and applies equally to the entire pipeline
of the retry.
"""
super(RetryAction, self).validate()
if not self.internal_pipeline:
raise LAVABug("Retry action %s needs to implement an internal pipeline" % self.name)
def run(self, connection, max_end_time, args=None):
if not connection:
raise LAVABug("%s started without a connection already in use" % self.name)
connection = super(BootloaderInterrupt, self).run(connection, max_end_time, args)
self.logger.debug("Changing prompt to '%s'", IPXE_BOOT_PROMPT)
# device is to be put into a reset state, either by issuing 'reboot' or power-cycle
connection.prompt_str = IPXE_BOOT_PROMPT
self.wait(connection)
connection.sendcontrol("b")
return connection
def run(self, connection, max_end_time, args=None):
connection = super(TestAdjuvant.FakeAdjuvant,
self).run(connection, max_end_time, args)
if not self.valid:
raise LAVABug("fakeadjuvant should be valid")
if self.data[self.key()]:
self.data[self.key()] = 'triggered'
if self.adjuvant:
self.data[self.key()] = 'base class trigger'
return connection
def run(self, connection, max_end_time, args=None):
connection = super(MenuConnect, self).run(connection, max_end_time, args)
if not connection:
raise LAVABug("%s needs a Connection")
connection.check_char = '\n'
connection.sendline('\n') # to catch the first prompt (remove for PDU?)
connection.prompt_str = self.parameters['prompts']
if hasattr(self.job.device, 'power_state') and self.job.device.power_state not in ['on', 'off']:
self.wait(connection)
return connection
def run(self, connection, max_end_time, args=None):
if not connection:
raise LAVABug("%s started without a connection already in use" % self.name)
connection = super(EnableVExpressMassStorage, self).run(connection, max_end_time, args)
# Issue command and check that you are returned to the prompt again
connection.sendline('%s\n' % self.mcc_cmd)
self.logger.debug("Changing prompt to '%s'", self.mcc_prompt)
connection.prompt_str = self.mcc_prompt
self.wait(connection)
return connection
def run(self, connection, max_end_time, args=None):
if not connection:
raise LAVABug("Called %s without an active Connection" % self.name)
if not self.valid or self.key() not in self.data:
return connection
if self.data[self.key()]:
self.adjuvant = True
self.logger.warning("Adjuvant %s required", self.name)
else:
self.logger.debug("Adjuvant %s skipped", self.name)
return connection
def _api_select(self, data, action=None):
if not data:
raise TestError("[%s] Protocol called without any data." % self.name)
if not action:
raise LAVABug('LXC protocol needs to be called from an action.')
for item in data:
if 'request' not in item:
raise LAVABug("[%s] Malformed protocol request data." % self.name)
if 'pre-os-command' in item['request']:
action.logger.info("[%s] Running pre OS command via protocol.", self.name)
command = action.job.device.pre_os_command
if not action.run_command(command.split(' '), allow_silent=True):
raise InfrastructureError("%s failed" % command)
continue
elif 'pre-power-command' in item['request']:
action.logger.info("[%s] Running pre-power-command via protocol.", self.name)
command = action.job.device.pre_power_command
if not action.run_command(command.split(' '), allow_silent=True):
raise InfrastructureError("%s failed" % command)
continue
else:
raise JobError("[%s] Unrecognised protocol request: %s" % (self.name, item))
def rmtree(directory):
"""
Wrapper around shutil.rmtree to remove a directory tree while ignoring most
errors.
If called on a symbolic link, this function will raise a LAVABug.
"""
# TODO: consider how to handle problems if the directory has already been removed -
# coding bugs may trigger this Runtime exception - implement before moving to production.
try:
shutil.rmtree(directory)
except OSError as exc:
raise LAVABug("Error when trying to remove '%s': %s"
% (directory, exc))
def listen_feedback(self, timeout):
"""
Listen to output and log as feedback
"""
if timeout < 0:
raise LAVABug("Invalid timeout value passed to listen_feedback()")
try:
self.raw_connection.logfile.is_feedback = True
return self.raw_connection.expect([pexpect.EOF, pexpect.TIMEOUT],
timeout=timeout)
except KeyboardInterrupt:
raise
finally:
self.raw_connection.logfile.is_feedback = False
def populate(self, parameters):
"""
Needs to take account of the deployment type / image type etc.
to determine which actions need to be added to the internal pipeline
as part of the deployment selection step.
"""
if not self.job:
raise LAVABug("No job object supplied to action")
# FIXME: not all mount operations will need these actions
self.internal_pipeline = Pipeline(parent=self,
job=self.job,
parameters=parameters)
self.internal_pipeline.add_action(OffsetAction(self.key))
# FIXME: LoopCheckAction and LoopMountAction should be in only one Action
self.internal_pipeline.add_action(LoopCheckAction(self.key))
self.internal_pipeline.add_action(LoopMountAction(self.key))
def run(self, connection, max_end_time, args=None):
if not connection:
raise LAVABug("%s started without a connection already in use" % self.name)
connection = super(EnterVExpressMCC, self).run(connection, max_end_time, args)
# Get possible prompts from device config
connection.prompt_str = [self.autorun_prompt, self.mcc_prompt]
self.logger.debug("Changing prompt to '%s'", connection.prompt_str)
index = self.wait(connection)
if connection.prompt_str[index] != self.mcc_prompt:
self.logger.debug('Autorun enabled: interrupting..')
connection.sendline('%s\n' % self.interrupt_char)
connection.prompt_str = self.mcc_prompt
self.wait(connection)
else:
self.logger.debug('Already at MCC prompt: autorun looks to be disabled')
return connection
def wait(self, max_end_time=None):
"""
Simple wait without sendling blank lines as that causes the menu
to advance without data which can cause blank entries and can cause
the menu to exit to an unrecognised prompt.
"""
if not max_end_time:
timeout = self.timeout.duration
else:
timeout = max_end_time - time.time()
if timeout < 0:
raise LAVABug("Invalid max_end_time value passed to wait()")
try:
return self.raw_connection.expect(self.prompt_str, timeout=timeout)
except (TestError, pexpect.TIMEOUT):
raise JobError("wait for prompt timed out")
except KeyboardInterrupt:
raise
def run(self, connection, max_end_time, args=None):
if not connection:
raise LAVABug("Called %s without an active Connection" % self.name)
if self.job.device.power_state is 'off' and self.job.device.power_command is not '': # power on action used instead
return connection
if self.job.device.power_state is 'on' and self.job.device.soft_reset_command is not '':
command = self.job.device['commands']['soft_reset']
if isinstance(command, list):
for cmd in command:
if not self.run_command(cmd.split(' '), allow_silent=True):
raise InfrastructureError("%s failed" % cmd)
else:
if not self.run_command(command.split(' '), allow_silent=True):
raise InfrastructureError("%s failed" % command)
self.results = {"success": self.job.device.power_state}
else:
connection = super(RebootDevice, self).run(connection,
max_end_time, args)
connection.prompt_str = self.parameters.get('parameters', {}).get(
'shutdown-message',
self.job.device.get_constant('shutdown-message'))
connection.timeout = self.connection_timeout
connection.sendline("reboot")
# FIXME: possibly deployment data, possibly separate actions, possibly adjuvants.
connection.sendline(
"reboot -n") # initramfs may require -n for *now*
connection.sendline(
"reboot -n -f"
) # initrd may require -n for *now* and -f for *force*
self.results = {"success": connection.prompt_str}
self.data[PDUReboot.key()] = False
reboot_prompt = self.get_namespace_data(action='uboot-retry',
label='bootloader_prompt',
key='prompt')
if reboot_prompt:
self.reboot_prompt = reboot_prompt
try:
self.wait(connection)
except TestError:
self.logger.info("Wait for prompt after soft reboot failed")
self.results = {'status': "failed"}
self.data[PDUReboot.key()] = True
connection.prompt_str = self.reboot_prompt
return connection
def run(self, connection, max_end_time, args=None):
if not connection:
raise LAVABug("%s started without a connection already in use" %
self.name)
connection = super(BootloaderInterrupt,
self).run(connection, max_end_time, args)
device_methods = self.job.device['actions']['boot']['methods']
interrupt_prompt = device_methods[self.type]['parameters'].get(
'interrupt_prompt',
self.job.device.get_constant('grub-autoboot-prompt'))
# interrupt_char can actually be a sequence of ASCII characters - sendline does not care.
interrupt_char = device_methods[self.type]['parameters'].get(
'interrupt_char',
self.job.device.get_constant('grub-interrupt-character'))
# device is to be put into a reset state, either by issuing 'reboot' or power-cycle
connection.prompt_str = interrupt_prompt
self.wait(connection)
connection.raw_connection.send(interrupt_char)
return connection
def run(self, connection, max_end_time, args=None):
connection = super(LoopCheckAction, self).run(connection, max_end_time,
args)
if not self.get_namespace_data(
action=self.name, label=self.key, key='available_loops'):
raise LAVABug("Unable to check available loop devices")
args = ['/sbin/losetup', '-a']
pro = self.run_command(args)
mounted_loops = len(pro.strip().split("\n")) if pro else 0
available_loops = self.get_namespace_data(action=self.name,
label=self.key,
key='available_loops')
# FIXME: we should retry as this can happen and be fixed automatically
# when one is unmounted
if mounted_loops >= available_loops:
raise InfrastructureError("Insufficient loopback devices?")
self.logger.debug("available loops: %s", available_loops)
self.logger.debug("mounted_loops: %s", mounted_loops)
return connection
