def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) mount_point = self.get_namespace_data(action="mount-vexpress-usbmsd", label="vexpress-fw", key="mount-point") if not os.path.exists(mount_point): raise InfrastructureError("Unable to locate mount point: %s" % mount_point) src_dir = self.get_namespace_data( action="extract-vexpress-recovery-image", label="file", key="recovery_image") if not os.path.exists(src_dir): raise InfrastructureError( "Unable to locate recovery image source directory: %s" % src_dir) self.logger.debug( "Removing existing recovery image from Versatile Express mass storage device.." ) try: remove_directory_contents(mount_point) except Exception: raise JobError("Failed to erase old recovery image") self.logger.debug( "Transferring new recovery image to Versatile Express mass storage device.." ) try: copy_directory_contents(src_dir, mount_point) except Exception: raise JobError("Failed to deploy recovery image to %s" % mount_point) return connection
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) device_path = "/dev/disk/by-%s/%s" % ( self.disk_identifier_type, self.disk_identifier, ) if not os.path.exists(device_path): raise InfrastructureError("Unable to find disk by %s %s" % (self.disk_identifier_type, device_path)) mount_point = "/mnt/%s" % self.disk_identifier if not os.path.exists(mount_point): try: self.logger.debug("Creating mount point '%s'", mount_point) os.makedirs(mount_point, 0o755) except OSError: raise InfrastructureError("Failed to create mount point %s" % mount_point) self.run_cmd( ["mount", device_path, mount_point], error_msg="Failed to mount device %s to %s" % (device_path, mount_point), ) self.set_namespace_data( action=self.name, label=self.namespace_label, key="mount-point", value=mount_point, ) return connection
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) dfu = self.job.device["actions"]["boot"]["methods"]["dfu"] reset_works = dfu.get("reset_works", True) implementation = dfu.get("implementation", "hardware") # Store the previous prompt_str to restore it afterward prompt_str = connection.prompt_str if implementation != "hardware": connection.prompt_str = self.job.device.get_constant( "dfu-download", prefix=implementation) for (index, dfu_command) in enumerate(self.exec_list): # add --reset for the last command (if reset works) if index + 1 == len(self.exec_list) and reset_works: dfu_command.extend(["--reset"]) dfu = " ".join(dfu_command) output = self.run_command(dfu.split(" ")) # Check the output as dfu-util can return 0 in case of errors. if output: if "No error condition is present\nDone!\n" not in output: raise InfrastructureError("command failed: %s" % dfu) else: raise InfrastructureError("command failed: %s" % dfu) # Wait only for non-hardware implementations # In fact, the booloader will print some strings when the transfert # is finished. if implementation != "hardware": connection.wait() # Restore the prompts connection.prompt_str = prompt_str return connection
def _api_select(self, data, action=None): if not data: raise TestError("[%s] Protocol called without any data." % self.name) if not action: raise LAVABug('LXC protocol needs to be called from an action.') for item in data: if 'request' not in item: raise LAVABug("[%s] Malformed protocol request data." % self.name) if 'pre-os-command' in item['request']: action.logger.info("[%s] Running pre OS command via protocol.", self.name) command = action.job.device.pre_os_command if not command: raise JobError("No pre OS command is defined for this device.") if not isinstance(command, list): command = [command] for cmd in command: if not action.run_command(cmd.split(' '), allow_silent=True): raise InfrastructureError("%s failed" % cmd) continue elif 'pre-power-command' in item['request']: action.logger.info("[%s] Running pre-power-command via protocol.", self.name) command = action.job.device.pre_power_command if not command: raise JobError("No pre power command is defined for this device.") if not isinstance(command, list): command = [command] for cmd in command: if not action.run_command(cmd.split(' '), allow_silent=True): raise InfrastructureError("%s failed" % cmd) continue else: raise JobError("[%s] Unrecognised protocol request: %s" % (self.name, item))
def binary_version(binary, flags="", pattern=""): """ Returns a string with the version of the binary by running it with the provided flags. If the output from running the binary is verbose and contains more than just the version number, a pattern can be provided for the function to parse the output to pick out the substring that contains the version number. """ # if binary is not absolute, fail. msg = "Unable to retrieve version of %s" % binary try: ver_str = (subprocess.check_output( (binary, flags), stderr=subprocess.STDOUT).strip().decode("utf-8", errors="replace")) if not ver_str: raise InfrastructureError(msg) except (subprocess.CalledProcessError, FileNotFoundError): raise InfrastructureError(msg) if pattern != "": p = re.compile(pattern) result = p.search(ver_str) if result is not None: ver_str = result.group(1) else: raise InfrastructureError(msg) return "%s, version %s" % (binary, ver_str)
def copy_in_overlay(image, root_partition, overlay): """ Mounts test image partition as specified by the test writer and extracts overlay at the root, if root_partition is None the image is handled as a filesystem instead of partitioned image. """ guest = guestfs.GuestFS(python_return_dict=True) guest.add_drive(image) _launch_guestfs(guest) if root_partition is not None: partitions = guest.list_partitions() if not partitions: raise InfrastructureError("Unable to prepare guestfs") guest_partition = partitions[root_partition] guest.mount(guest_partition, "/") else: devices = guest.list_devices() if not devices: raise InfrastructureError("Unable to prepare guestfs") guest.mount(devices[0], "/") # FIXME: max message length issues when using tar_in # on tar.gz. Works fine with tar so decompressing # overlay first. if os.path.exists(overlay[:-3]): os.unlink(overlay[:-3]) decompressed_overlay = decompress_file(overlay, "gz") guest.tar_in(decompressed_overlay, "/") if root_partition is not None: guest.umount(guest_partition) else: guest.umount(devices[0])
def reader(self): res = None try: # FIXME: When requests 3.0 is released, use the enforce_content_length # parameter to raise an exception the file is not fully downloaded headers = None if self.params and "headers" in self.params: headers = self.params["headers"] res = requests_retry().get( self.url.geturl(), allow_redirects=True, stream=True, headers=headers, timeout=HTTP_DOWNLOAD_TIMEOUT, ) if res.status_code != requests.codes.OK: # This is an Infrastructure error because the validate function # checked that the file does exist. raise InfrastructureError("Unable to download '%s'" % (self.url.geturl())) for buff in res.iter_content(HTTP_DOWNLOAD_CHUNK_SIZE): yield buff except requests.RequestException as exc: raise InfrastructureError("Unable to download '%s': %s" % (self.url.geturl(), str(exc))) finally: if res is not None: res.close()
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) # copy recovery image to a temporary directory and unpack recovery_image = self.get_namespace_data(action="download-action", label=self.param_key, key="file") recovery_image_dir = self.mkdtemp() shutil.copy(recovery_image, recovery_image_dir) tmp_recovery_image = os.path.join(recovery_image_dir, os.path.basename(recovery_image)) if os.path.isfile(tmp_recovery_image): if self.compression == "zip": decompress_file(tmp_recovery_image, self.compression) elif self.compression == "gz": untar_file(tmp_recovery_image, recovery_image_dir) else: raise InfrastructureError( "Unsupported compression for VExpress recovery: %s" % self.compression) os.remove(tmp_recovery_image) self.set_namespace_data( action="extract-vexpress-recovery-image", label="file", key=self.file_key, value=recovery_image_dir, ) self.logger.debug("Extracted %s to %s", self.file_key, recovery_image_dir) else: raise InfrastructureError("Unable to decompress recovery image") return connection
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) device_path = "/dev/disk/by-label/%s" % self.microsd_fs_label try: os.path.realpath(device_path) except OSError: raise InfrastructureError("Unable to find disk by label %s" % device_path) mount_point = "/mnt/%s" % self.microsd_fs_label if not os.path.exists(mount_point): try: self.logger.debug("Creating mount point '%s'", mount_point) os.makedirs(mount_point, 0o755) except OSError: raise InfrastructureError("Failed to create mount point %s" % mount_point) mount_cmd = ["mount", device_path, mount_point] if not self.run_command(mount_cmd, allow_silent=True): raise InfrastructureError("Failed to mount device %s to %s" % (device_path, mount_point)) self.set_namespace_data(action=self.name, label="vexpress-fw", key="mount-point", value=mount_point) return connection
def run(self, connection, max_end_time): connection = self.get_namespace_data(action='shared', label='shared', key='connection', deepcopy=False) connection = super().run(connection, max_end_time) count = 1 for dfu_command in self.exec_list: if count == (len(self.exec_list)): if self.job.device['actions']['boot']['methods']['dfu'].get( 'reset_works', True): dfu_command.extend(['--reset']) dfu = ' '.join(dfu_command) output = self.run_command(dfu.split(' ')) if output: if "No error condition is present\nDone!\n" not in output: raise InfrastructureError("command failed: %s" % dfu) else: raise InfrastructureError("command failed: %s" % dfu) count += 1 self.set_namespace_data(action='shared', label='shared', key='connection', value=connection) return connection
def which(path, match=os.path.isfile): exefile = _which_check(path, match) if not exefile: raise InfrastructureError("Cannot find command '%s' in $PATH" % path) if os.stat(exefile).st_mode & S_IXUSR != S_IXUSR: raise InfrastructureError("Cannot execute '%s' at '%s'" % (path, exefile)) return exefile
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) if not self.job.device.hard_reset_command: raise InfrastructureError("Hard reset required but not defined.") command = self.job.device.hard_reset_command if not isinstance(command, list): command = [command] for cmd in command: if not self.run_command(cmd.split(' '), allow_silent=True): raise InfrastructureError("%s failed" % cmd) self.results = {'status': 'success'} return connection
def append_dtb(self, kernel_file, dtb_file, dest_file): self.logger.info("Appending %s to %s", dtb_file, kernel_file) # Can't use cat here because it will be called with subprocess.check_output that catches stdout cmd = ["dd", "if=%s" % kernel_file, "of=%s" % dest_file] cmd2 = [ "dd", "if=%s" % dtb_file, "of=%s" % dest_file, "oflag=append", "conv=notrunc" ] if not self.run_command(cmd): raise InfrastructureError("DTB appending failed") if not self.run_command(cmd2): raise InfrastructureError("DTB appending failed")
def binary_version(binary, flags=""): """ Returns a string with the version of the binary by running it with the provided flags. """ # if binary is not absolute, fail. msg = "Unable to retrieve version of %s" % binary try: ver_str = (subprocess.check_output( (binary, flags)).strip().decode("utf-8", errors="replace")) if not ver_str: raise InfrastructureError(msg) except (subprocess.CalledProcessError, FileNotFoundError): raise InfrastructureError(msg) return "%s, version %s" % (binary, ver_str)
def run(self, connection, max_end_time): if not connection: raise LAVABug("%s started without a connection already in use" % self.name) connection = super().run(connection, max_end_time) # Get possible prompts from device config prompt_list = [ self.autorun_prompt, self.mcc_prompt, self.mcc_reset_msg ] connection.prompt_str = prompt_list self.logger.debug("Changing prompt to '%s'", connection.prompt_str) index = self.wait(connection) # Interrupt autorun if enabled if connection.prompt_str[index] == self.autorun_prompt: self.logger.debug("Autorun enabled: interrupting..") connection.sendline(self.interrupt_char) connection.prompt_str = [self.mcc_prompt, self.mcc_reset_msg] index = self.wait(connection) elif connection.prompt_str[index] == self.mcc_prompt: self.logger.debug( "Already at MCC prompt: autorun looks to be disabled") # Check that mcc_reset_msg hasn't been received if connection.prompt_str[index] == self.mcc_reset_msg: raise InfrastructureError("MCC: Unable to interrupt auto-run") return connection
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) mount_point = self.get_namespace_data(action='mount-vexpress-usbmsd', label='vexpress-fw', key='mount-point') if not self.run_command(["umount", mount_point], allow_silent=True): raise InfrastructureError("Failed to unmount device %s" % mount_point) return connection
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) if not connection: raise InfrastructureError("Connection closed") # Get substitutions from bootloader-overlay substitutions = self.get_namespace_data( action="bootloader-overlay", label="u-boot", key="substitutions" ) if substitutions is None: substitutions = {} # Loop on all scripts for script in self.parameters["interactive"]: start = time.time() result = { "definition": "lava", "case": "%s_%s" % (self.parameters["stage"], script["name"]), "result": "fail", } try: with connection.test_connection() as test_connection: self.run_script(test_connection, script, substitutions) result["result"] = "pass" finally: # Log the current script result (even in case of error) result["duration"] = "%.02f" % (time.time() - start) self.logger.results(result) # Set the connection prompts connection.prompt_str = script["prompts"] return connection
def copy_overlay_to_sparse_fs(image, overlay): """copy_overlay_to_sparse_fs Only copies the overlay to an image which has already been converted from sparse. """ logger = logging.getLogger("dispatcher") guest = guestfs.GuestFS(python_return_dict=True) guest.add_drive(image) _launch_guestfs(guest) devices = guest.list_devices() if not devices: raise InfrastructureError("Unable to prepare guestfs") guest.mount(devices[0], "/") # FIXME: max message length issues when using tar_in # on tar.gz. Works fine with tar so decompressing # overlay first. if os.path.exists(overlay[:-3]): os.unlink(overlay[:-3]) decompressed_overlay = decompress_file(overlay, "gz") guest.tar_in(decompressed_overlay, "/") # Check if we have space left on the mounted image. output = guest.df() logger.debug(output) _, _, _, available, percent, _ = output.split("\n")[1].split() guest.umount(devices[0]) if int(available) == 0 or percent == "100%": raise JobError("No space in image after applying overlay: %s" % image)
def DeviceFilter(*args, **kwargs): for klass in [DeviceFilterCGroupsV1, DeviceFilterCGroupsV2]: if klass.detect(): return klass(*args, **kwargs) raise InfrastructureError( "Neither cgroups v1 nor v2 detected; can't share device with docker container" )
def share_device_with_container(options, setup_logger=None): data = find_mapping(options) if not data: return if setup_logger: setup_logger(data["logging_info"]) logger = logging.getLogger("dispatcher") container = data["container"] device = "/dev/" + options.device if not os.path.exists(device): logger.warning( "Can't share {device}: file not found".format(device=device)) return container_type = data["container_type"] if container_type == "lxc": share_device_with_container_lxc(container, device) else: raise InfrastructureError('Unsupported container type: "%s"' % container_type) logger.info( "Sharing {device} with {container_type} container {container}".format( device=device, container_type=data["container_type"], container=data["container"], ))
def dispatcher_ip(dispatcher_config, protocol=None): """ Retrieves the IP address of the interface associated with the current default gateway. :param protocol: 'http', 'tftp' or 'nfs' """ if protocol: if protocol not in VALID_DISPATCHER_IP_PROTOCOLS: raise LAVABug("protocol should be one of %s" % VALID_DISPATCHER_IP_PROTOCOLS) with contextlib.suppress(KeyError, TypeError): return dispatcher_config["dispatcher_%s_ip" % protocol] with contextlib.suppress(KeyError, TypeError): return dispatcher_config["dispatcher_ip"] gateways = netifaces.gateways() if "default" not in gateways: raise InfrastructureError( "Unable to find dispatcher 'default' gateway") iface = gateways["default"][netifaces.AF_INET][1] iface_addr = None try: addr = netifaces.ifaddresses(iface) iface_addr = addr[netifaces.AF_INET][0]["addr"] except KeyError: # TODO: This only handles first alias interface can be extended # to review all alias interfaces. addr = netifaces.ifaddresses(iface + ":0") iface_addr = addr[netifaces.AF_INET][0]["addr"] return iface_addr
def create_tarfile(indir, outfile): try: with tarfile.open(outfile, "w") as tar: tar.add(indir) except tarfile.TarError as exc: raise InfrastructureError("Unable to create lava overlay tarball: %s" % exc)
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) params = { label: self.get_namespace_data(action='download-action', label=label, key='file') for label in ['kernel', 'dtb', 'ramdisk'] } fit_path = os.path.join(os.path.dirname(params['kernel']), 'image.itb') params.update({ 'arch': self.deploy_params['mkimage_arch'], 'load_addr': self.device_params['load_address'], 'fit_path': fit_path, }) ramdisk_with_overlay = self.get_namespace_data( action='compress-ramdisk', label='file', key='full-path') if ramdisk_with_overlay: params['ramdisk'] = ramdisk_with_overlay cmd = self._make_mkimage_command(params) if not self.run_command(cmd): raise InfrastructureError("FIT image creation failed") kernel_tftp = self.get_namespace_data(action='download-action', label='file', key='kernel') fit_tftp = os.path.join(os.path.dirname(kernel_tftp), 'image.itb') self.set_namespace_data(action=self.name, label='file', key='fit', value=fit_tftp) return connection
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) params = { label: self.get_namespace_data(action="download-action", label=label, key="file") for label in ["kernel", "dtb", "ramdisk"] } fit_path = os.path.join(os.path.dirname(params["kernel"]), "image.itb") params.update({ "arch": self.deploy_params["mkimage_arch"], "load_addr": self.device_params["load_address"], "fit_path": fit_path, }) ramdisk_with_overlay = self.get_namespace_data( action="compress-ramdisk", label="file", key="full-path") if ramdisk_with_overlay: params["ramdisk"] = ramdisk_with_overlay cmd = self._make_mkimage_command(params) if not self.run_command(cmd): raise InfrastructureError("FIT image creation failed") kernel_tftp = self.get_namespace_data(action="download-action", label="file", key="kernel") fit_tftp = os.path.join(os.path.dirname(kernel_tftp), "image.itb") self.set_namespace_data(action=self.name, label="file", key="fit", value=fit_tftp) return connection
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) if not connection: raise InfrastructureError("Connection closed") for monitor in self.parameters["monitors"]: self.test_suite_name = monitor["name"] self.fixupdict = monitor.get("fixupdict") # pattern order is important because we want to match the end before # it can possibly get confused with a test result self.patterns = OrderedDict() self.patterns["eof"] = pexpect.EOF self.patterns["timeout"] = pexpect.TIMEOUT self.patterns["end"] = monitor["end"] self.patterns["test_result"] = monitor["pattern"] # Find the start string before parsing any output. self.logger.info("Waiting for start message: %s", monitor["start"]) connection.prompt_str = monitor["start"] connection.wait() self.logger.info( "ok: start string found, lava test monitoring started") with connection.test_connection() as test_connection: while self._keep_running(test_connection, timeout=test_connection.timeout): pass return connection
def run(self, connection, max_end_time): lxc_active = any([ protocol for protocol in self.job.protocols if protocol.name == LxcProtocol.name ]) if self.job.device.pre_os_command and not lxc_active: self.logger.info("Running pre OS command.") command = self.job.device.pre_os_command if not self.run_command(command.split(" "), allow_silent=True): raise InfrastructureError("%s failed" % command) if not connection: self.logger.debug("Existing connection in %s", self.name) return connection connection.prompt_str = self.selector.prompt connection.raw_connection.linesep = self.line_sep self.logger.debug("Looking for %s", self.selector.prompt) self.wait(connection) connection = super().run(connection, max_end_time) if self.boot_message: self.logger.debug("Looking for %s", self.boot_message) connection.prompt_str = self.boot_message self.wait(connection) self.set_namespace_data(action="shared", label="shared", key="connection", value=connection) return connection
def run(self, connection, max_end_time): output = os.path.join(self.mkdtemp(), "overlay-%s.tar.gz" % self.level) location = self.get_namespace_data(action='test', label='shared', key='location') lava_test_results_dir = self.get_namespace_data( action='test', label='results', key='lava_test_results_dir') self.set_namespace_data(action='test', label='shared', key='output', value=output) if not location: raise LAVABug("Missing lava overlay location") if not os.path.exists(location): raise LAVABug("Unable to find overlay location") if not self.valid: self.logger.error(self.errors) return connection connection = super().run(connection, max_end_time) with chdir(location): try: with tarfile.open(output, "w:gz") as tar: tar.add(".%s" % lava_test_results_dir) # ssh authorization support if os.path.exists('./root/'): tar.add(".%s" % '/root/') except tarfile.TarError as exc: raise InfrastructureError( "Unable to create lava overlay tarball: %s" % exc) self.set_namespace_data(action=self.name, label='output', key='file', value=output) return connection
def run(self, connection, max_end_time): # Prompts commonly include # - when logging such strings, # use lazy logging or the string will not be quoted correctly. if self.booting: kernel_start_message = self.parameters.get("parameters", {}).get( "kernel-start-message", self.job.device.get_constant("kernel-start-message"), ) if kernel_start_message: connection.prompt_str = [kernel_start_message] if self.params and self.params.get("boot_message"): self.logger.warning( "boot_message is being deprecated in favour of kernel-start-message in constants" ) connection.prompt_str = [self.params.get("boot_message")] error_messages = self.job.device.get_constant("error-messages", prefix=self.method, missing_ok=True) if error_messages: if isinstance(connection.prompt_str, str): connection.prompt_str = [connection.prompt_str] connection.prompt_str = connection.prompt_str + error_messages if kernel_start_message: res = self.wait(connection) if res != 0: msg = "matched a bootloader error message: '%s' (%d)" % ( connection.prompt_str[res], res, ) raise InfrastructureError(msg) connection = super().run(connection, max_end_time) return connection
def dispatcher_gateway(): """ Retrieves the IP address of the current default gateway. """ gateways = netifaces.gateways() if "default" not in gateways: raise InfrastructureError("Unable to find default gateway") return gateways["default"][netifaces.AF_INET][0]
def _launch_guestfs(guest): # Launch guestfs and raise an InfrastructureError if needed try: guest.launch() except RuntimeError as exc: logger = logging.getLogger("dispatcher") logger.exception(str(exc)) raise InfrastructureError("Unable to start libguestfs")