def test_run_architecture_check_failure(mocker): def results(cmd, *args, **kwargs): if cmd == ["arch"]: return "aarch64\n" elif cmd == ["docker", "run", "--rm", "myimage", "arch"]: return "x86_64\n" else: raise RuntimeError(f"Unexpected mock call: {cmd}") check_output = mocker.patch("subprocess.check_output", side_effect=results) check_call = mocker.patch("subprocess.check_call") getLogger = mocker.patch("logging.getLogger") logger = getLogger.return_value docker = DockerRun("myimage") docker.run("date") check_output.assert_any_call(["arch"], text=True) check_output.assert_any_call( ["docker", "run", "--rm", "myimage", "arch"], text=True ) check_call.assert_called() getLogger.assert_called_with("dispatcher") logger.warning.assert_called()
def test_from_parameters_image(mocker): job = mocker.MagicMock() assert DockerRun.from_parameters({"image": "foo"}, job).image == "foo" assert not DockerRun.from_parameters({"image": "foo"}, job).__local__ assert DockerRun.from_parameters({ "image": "foo", "local": True }, job).__local__
def test_run_architecture_check_success(mocker): check_output = mocker.patch("subprocess.check_output", return_value="xyz\n") check_call = mocker.patch("subprocess.check_call") getLogger = mocker.patch("logging.getLogger") docker = DockerRun("myimage") docker.run("echo") # no crash = success check_call.assert_called_with(["docker", "run", "--rm", "myimage", "echo"]) getLogger.assert_not_called()
def test_run_with_action(mocker): check_arch = mocker.patch( "lava_dispatcher.utils.docker.DockerRun.__check_image_arch__") action = mocker.MagicMock() docker = DockerRun("myimage") docker.run("date", action=action) check_arch.assert_called() action.run_cmd.assert_has_calls([ mocker.call(["docker", "pull", "myimage"]), mocker.call(["docker", "run", "--rm", "myimage", "date"]), ])
def run(self, connection, max_end_time): job_id = self.job.job_id script = ["#!/bin/sh", "exec 2>&1", "set -ex"] # Export data generated during run of the Pipeline like NFS settings if self.job.device: for key in self.job.device["dynamic_data"]: script.append("export %s='%s'" % (key, self.job.device["dynamic_data"][key])) script = script + self.steps script = "\n".join(script) + "\n" scriptfile = self.path / "postprocess.sh" scriptfile.write_text(script) scriptfile.chmod(0o755) docker = DockerRun.from_parameters(self.docker_parameters, self.job) docker.add_device("/dev/kvm", skip_missing=True) docker.bind_mount(self.path, LAVA_DOWNLOADS) docker.workdir(LAVA_DOWNLOADS) docker.run(f"{LAVA_DOWNLOADS}/postprocess.sh", action=self) return connection
def get_command_prefix(self): docker = DockerRun(self.image) for device in self.__get_device_nodes__(): docker.add_device(device) for f in self.copied_files: docker.bind_mount(f) return docker.cmdline()
def test_from_parameters_name_network(mocker): job = mocker.MagicMock() job.job_id = "123" docker_run = DockerRun.from_parameters( { "image": "foo", "container_name": "foocontainer", "network_from": "othercontainer", }, job, ) assert docker_run.__name__ == "foocontainer-lava-123" assert docker_run.__network__ == "othercontainer"
def get_command_prefix(self): docker = DockerRun.from_parameters(self.params) docker.add_docker_options(*self.docker_options) docker.add_docker_run_options(*self.docker_run_options) for device in self.__get_device_nodes__(): docker.add_device(device) if not self.docker_options: for f in self.copied_files: docker.bind_mount(f) return docker.cmdline()
def test_run_with_local_image_does_not_pull(mocker): mocker.patch("lava_dispatcher.utils.docker.DockerRun.__check_image_arch__") docker = DockerRun("myimage") docker.local(True) action = mocker.MagicMock() docker.run("date", action=action) action.run_cmd.assert_has_calls([ mocker.call(["docker", "image", "inspect", mocker.ANY, "myimage"]), mocker.call(["docker", "run", "--rm", "--init", "myimage", "date"]), ])
def run(self, connection, max_end_time): job_id = self.job.job_id script = ["#!/bin/sh", "exec 2>&1", "set -ex"] + self.steps script = "\n".join(script) + "\n" scriptfile = self.path / "postprocess.sh" scriptfile.write_text(script) scriptfile.chmod(0o755) docker = DockerRun.from_parameters(self.docker_parameters) docker.add_device("/dev/kvm", skip_missing=True) docker.bind_mount(self.path, LAVA_DOWNLOADS) docker.hostname("lava") docker.workdir(LAVA_DOWNLOADS) docker.run(f"{LAVA_DOWNLOADS}/postprocess.sh", action=self) return connection
def get_raw_version(self, architecture): if "docker" in self.parameters: docker = DockerRun.from_parameters(self.parameters["docker"], self.job) docker.run(*shlex.split("qemu-system-%s --version" % self.get_qemu_arch(architecture)), action=self) return True ver_strs = subprocess.check_output( ("qemu-system-%s" % architecture, "--version")) # line is QEMU emulator version xxxx ver_str = ver_strs.split()[3].decode("utf-8", errors="replace") arch_str = (subprocess.check_output( ("uname", "-m")).strip().decode("utf-8", errors="replace")) self.qemu_data = { "qemu_version": ver_str, "host_arch": arch_str, "job_arch": architecture, } self.logger.info( "qemu, installed at version: %s, host architecture: %s", ver_str, arch_str) return True
def test_wait(mocker): docker = DockerRun("myimage") docker.name("foobar") sleep = mocker.patch("time.sleep") inspect = mocker.patch( "subprocess.check_call", side_effect=[ subprocess.CalledProcessError( 1, ["docker", "inspect", "--format=.", "foobar"]), None, ], ) docker.wait() call = mocker.call( ["docker", "inspect", mocker.ANY, "foobar"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) inspect.assert_has_calls([call, call]) sleep.assert_called_once()
def run(): return DockerRun("foobar")
def test_from_parameters(): assert DockerRun.from_parameters({"image": "foo"}).image == "foo" assert not DockerRun.from_parameters({"image": "foo"}).__local__ assert DockerRun.from_parameters({"image": "foo", "local": True}).__local__
def run(self, connection, max_end_time): # obtain lava overlay # start container # create USB device mapping to container # connect to container, and run lava-test-shell over it location = self.get_namespace_data(action="test", label="shared", key="location") overlay = self.get_namespace_data( action="test", label="results", key="lava_test_results_dir").strip("/") container = "lava-docker-test-shell-%s-%s" % (self.job.job_id, self.level) docker = DockerRun.from_parameters(self.parameters["docker"], self.job) docker.prepare() docker.bind_mount(os.path.join(location, overlay), "/" + overlay) docker_method_conf = (self.job.device["actions"].get("test", {}).get( "methods", {}).get("docker", {})) # Preprocess docker option list, to better support partial # overriding of them via device dict: # 1. Filter out None, to make it easier to template # YAML syntactic lists with Jinja2: # '- {{ some_opt_from_device_dict }}' # (if not default, will be set to None). # 2. Flatten sublists, `- ['--opt1', '--opt2']`. def preproc_opts(opts): res = [] for o in opts: if o is None: continue elif isinstance(o, list): res += o else: res.append(o) return res if "global_options" in docker_method_conf: docker.add_docker_options( *preproc_opts(docker_method_conf["global_options"])) if "options" in docker_method_conf: docker.add_docker_run_options( *preproc_opts(docker_method_conf["options"])) namespace = self.parameters.get("downloads-namespace", self.parameters.get("namespace")) if namespace: downloads_dir = pathlib.Path( self.job.tmp_dir) / "downloads" / namespace if downloads_dir.exists(): docker.bind_mount(downloads_dir, LAVA_DOWNLOADS) for bind_mount in self.test_docker_bind_mounts: read_only = True if len(bind_mount) == 2 else False docker.bind_mount(bind_mount[0], bind_mount[1], read_only) docker.interactive() docker.tty() docker.name(container) docker.environment("PS1", "docker-test-shell:$ ") docker_cmd = docker.cmdline("bash", "--norc", "-i") cmd = " ".join([shlex.quote(s) for s in docker_cmd]) self.logger.debug("Starting docker test shell container: %s" % cmd) shell = ShellCommand(cmd, self.timeout, logger=self.logger) shell_connection = ShellSession(self.job, shell) shell_connection.prompt_str = "docker-test-shell:" self.__set_connection__(shell_connection) self.add_device_container_mappings(container, "docker") devices = get_udev_devices(device_info=self.device_info, logger=self.logger, required=False) docker.wait(shell) # share all the devices as there isn't a 1:1 relationship between # the trigger and actual sharing of the devices for dev in devices: if not os.path.islink(dev): self.trigger_share_device_with_container(dev) for dev in devices: docker.wait_file(dev) try: super().run(shell_connection, max_end_time) finally: # finish the container shell_connection.finalise() docker.destroy() # return the original connection untouched self.__set_connection__(connection) return connection
def run(self, connection, max_end_time): """ CommandRunner expects a pexpect.spawn connection which is the return value of target.device.power_on executed by boot in the old dispatcher. In the new pipeline, the pexpect.spawn is a ShellCommand and the connection is a ShellSession. CommandRunner inside the ShellSession turns the ShellCommand into a runner which the ShellSession uses via ShellSession.run() to run commands issued *after* the device has booted. pexpect.spawn is one of the raw_connection objects for a Connection class. """ if connection: ns_connection = self.get_namespace_data( action="shared", label="shared", key="connection", deepcopy=False ) if connection == ns_connection: connection.finalise() self.sub_command = self.base_sub_command.copy() # Generate the sub command substitutions = {} for label in self.get_namespace_keys("download-action"): if label in ["offset", "available_loops", "uefi", "nfsrootfs"]: continue image_arg = self.get_namespace_data( action="download-action", label=label, key="image_arg" ) action_arg = self.get_namespace_data( action="download-action", label=label, key="file" ) if image_arg is not None: substitutions["{%s}" % label] = action_arg substitutions["{NFS_SERVER_IP}"] = dispatcher_ip( self.job.parameters["dispatcher"], "nfs" ) self.sub_command.extend(substitute(self.commands, substitutions)) uefi_dir = self.get_namespace_data( action="deployimages", label="image", key="uefi_dir" ) if uefi_dir: self.sub_command.extend(["-L", uefi_dir, "-monitor", "none"]) # initialise the first Connection object, a command line shell into the running QEMU. self.results = self.qemu_data guest = self.get_namespace_data( action="apply-overlay-guest", label="guest", key="filename" ) # check for NFS if "qemu-nfs" == self.parameters["method"]: self.logger.debug("Adding NFS arguments to kernel command line.") root_dir = self.get_namespace_data( action="extract-rootfs", label="file", key="nfsroot" ) substitutions["{NFSROOTFS}"] = root_dir params = self.methods["qemu-nfs"]["parameters"]["append"] # console=ttyAMA0 root=/dev/nfs nfsroot=10.3.2.1:/var/lib/lava/dispatcher/tmp/dirname,tcp,hard,intr ip=dhcp append = [ "console=%s" % params["console"], "root=/dev/nfs", "%s rw" % substitute([params["nfsrootargs"]], substitutions)[0], "%s" % params["ipargs"], ] self.sub_command.append("--append") self.sub_command.append('"%s"' % " ".join(append)) elif guest: self.logger.info("Extending command line for qcow2 test overlay") # interface is ide by default in qemu interface = self.job.device["actions"]["deploy"]["methods"]["image"][ "parameters" ]["guest"].get("interface", "ide") driveid = self.job.device["actions"]["deploy"]["methods"]["image"][ "parameters" ]["guest"].get("driveid", "lavatest") self.sub_command.append( "-drive format=qcow2,file=%s,media=disk,if=%s,id=%s" % (os.path.realpath(guest), interface, driveid) ) # push the mount operation to the test shell pre-command to be run # before the test shell tries to execute. shell_precommand_list = [] mountpoint = self.get_namespace_data( action="test", label="results", key="lava_test_results_dir" ) uuid = "/dev/disk/by-uuid/%s" % self.get_namespace_data( action="apply-overlay-guest", label="guest", key="UUID" ) shell_precommand_list.append("mkdir %s" % mountpoint) # prepare_guestfs always uses ext2 shell_precommand_list.append("mount %s -t ext2 %s" % (uuid, mountpoint)) # debug line to show the effect of the mount operation # also allows time for kernel messages from the mount operation to be processed. shell_precommand_list.append("ls -la %s/bin/lava-test-runner" % mountpoint) self.set_namespace_data( action="test", label="lava-test-shell", key="pre-command-list", value=shell_precommand_list, ) if "docker" in self.parameters: docker = DockerRun(self.parameters["docker"]["image"]) docker.interactive() docker.tty() docker.bind_mount(DISPATCHER_DOWNLOAD_DIR) docker.add_device("/dev/kvm", skip_missing=True) args = [] if "binary" in self.parameters["docker"]: args.append(self.parameters["docker"]["binary"]) self.sub_command[0] = " ".join(docker.cmdline(*args)) self.logger.info("Boot command: %s", " ".join(self.sub_command)) shell = self.shell_class( " ".join(self.sub_command), self.timeout, logger=self.logger ) if shell.exitstatus: raise JobError( "%s command exited %d: %s" % (self.sub_command, shell.exitstatus, shell.readlines()) ) self.logger.debug("started a shell command") shell_connection = self.session_class(self.job, shell) shell_connection = super().run(shell_connection, max_end_time) self.set_namespace_data( action="shared", label="shared", key="connection", value=shell_connection ) return shell_connection
def key(self): docker = DockerRun.from_parameters(self.params) return docker.image
def test_from_parameters_suffix(mocker): job = mocker.MagicMock() job.job_id = "123" docker_run = DockerRun.from_parameters({"image": "foo"}, job) assert docker_run.__suffix__ == "-lava-123"
def run(self, connection, max_end_time): # obtain lava overlay # start container # create USB device mapping to container # connect to container, and run lava-test-shell over it location = self.get_namespace_data(action="test", label="shared", key="location") overlay = self.get_namespace_data( action="test", label="results", key="lava_test_results_dir").strip("/") image = self.parameters["docker"]["image"] container = "lava-docker-test-shell-%s-%s" % (self.job.job_id, self.level) board_id = self.get_board_id() device_info = {"board_id": board_id} add_device_container_mapping( job_id=self.job.job_id, device_info=device_info, container=container, container_type="docker", logging_info=self.get_logging_info(), ) docker = DockerRun(image) docker.bind_mount(os.path.join(location, overlay), "/" + overlay) docker.interactive() docker.hostname("lava") docker.name(container) docker.environment("PS1", "docker-test-shell:$ ") if self.wait_for_device: devices = get_udev_devices(device_info=[device_info]) for dev in devices: docker.add_device(dev) docker_cmd = docker.cmdline("bash", "--norc", "-i") cmd = " ".join([shlex.quote(s) for s in docker_cmd]) self.logger.debug("Starting docker test shell container: %s" % cmd) shell = ShellCommand(cmd, self.timeout, logger=self.logger) shell_connection = ShellSession(self.job, shell) shell_connection.prompt_str = "docker-test-shell:" self.__set_connection__(shell_connection) super().run(shell_connection, max_end_time) # finish the container shell_connection.finalise() # return the original connection untouched self.__set_connection__(connection) return connection
def key(self): docker = DockerRun.from_parameters(self.params, self.action.job) return docker.image
def validate(self): docker = DockerRun.from_parameters(self.params, self.action.job) docker.prepare(self.action)
def run(self, connection, max_end_time): # obtain lava overlay # start container # create USB device mapping to container # connect to container, and run lava-test-shell over it location = self.get_namespace_data(action="test", label="shared", key="location") overlay = self.get_namespace_data( action="test", label="results", key="lava_test_results_dir").strip("/") container = "lava-docker-test-shell-%s-%s" % (self.job.job_id, self.level) docker = DockerRun.from_parameters(self.parameters["docker"]) docker.prepare() docker.bind_mount(os.path.join(location, overlay), "/" + overlay) namespace = self.parameters.get("downloads-namespace", self.parameters.get("namespace")) if namespace: downloads_dir = pathlib.Path( self.job.tmp_dir) / "downloads" / namespace if downloads_dir.exists(): docker.bind_mount(downloads_dir, LAVA_DOWNLOADS) for bind_mount in self.test_docker_bind_mounts: read_only = True if len(bind_mount) == 2 else False docker.bind_mount(bind_mount[0], bind_mount[1], read_only) docker.interactive() docker.tty() docker.hostname("lava") docker.name(container) docker.environment("PS1", "docker-test-shell:$ ") docker_cmd = docker.cmdline("bash", "--norc", "-i") cmd = " ".join([shlex.quote(s) for s in docker_cmd]) self.logger.debug("Starting docker test shell container: %s" % cmd) shell = ShellCommand(cmd, self.timeout, logger=self.logger) shell_connection = ShellSession(self.job, shell) shell_connection.prompt_str = "docker-test-shell:" self.__set_connection__(shell_connection) self.add_device_container_mappings(container, "docker") devices = get_udev_devices(device_info=self.device_info, logger=self.logger, required=False) docker.wait() # share all the devices as there isn't a 1:1 relationship between # the trigger and actual sharing of the devices for dev in devices: if not os.path.islink(dev): self.trigger_share_device_with_container(dev) for dev in devices: docker.wait_file(dev) try: super().run(shell_connection, max_end_time) finally: # finish the container shell_connection.finalise() docker.destroy() # return the original connection untouched self.__set_connection__(connection) return connection