def _pip_action(action, package): if action not in ["install", "uninstall"]: raise ValueError(f"{action} is invalid value for action") cmd = [sys.executable, "-m", "pip", action] if action == "uninstall": cmd += ["-y"] cmd += [package] logger.info("Installing requirements with command: %s", cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out_stream, err_stream = process.communicate() out_stream = out_stream.decode("utf-8") err_stream = err_stream.decode("utf-8") logger.debug("Return code is %s", process.returncode) logger.debug(out_stream) logger.debug(err_stream)
def init_vagrant(self, vagrant_file): """ Creates a Vagrantfile in the target dir, with only the base image pulled. Copies the runner script to the directory so it's accessible from the VM. """ if self.inherit_image: image_name, image_tag = str(self.inherit_image).split(":") else: image_name = self.get_arca_base_name() image_tag = self.get_python_base_tag(self.get_python_version()) logger.info("Creating Vagrantfile located in %s, base image %s:%s", vagrant_file, image_name, image_tag) repos_dir = (Path(self._arca.base_dir) / 'repos').resolve() vagrant_file.parent.mkdir(exist_ok=True, parents=True) vagrant_file.write_text(dedent(f""" # -*- mode: ruby -*- # vi: set ft=ruby : Vagrant.configure("2") do |config| config.vm.box = "{self.box}" config.ssh.insert_key = true config.vm.provision "docker" do |d| d.pull_images "{image_name}:{image_tag}" end config.vm.synced_folder ".", "/vagrant" config.vm.synced_folder "{repos_dir}", "/srv/repos" config.vm.provider "{self.provider}" end """)) (vagrant_file.parent / "runner.py").write_text(self.RUNNER.read_text())
def push_to_registry(self, image, image_tag: str): """ Pushes a local image to a registry based on the ``use_registry_name`` setting. :type image: docker.models.images.Image :raise PushToRegistryError: If the push fails. """ # already tagged, so it's already pushed if f"{self.use_registry_name}:{image_tag}" in image.tags: return image.tag(self.use_registry_name, image_tag) result = self.client.images.push(self.use_registry_name, image_tag) result = result.strip() # remove empty line at the end of output # the last can have one of two outputs, either # {"progressDetail":{},"aux":{"Tag":"<tag>","Digest":"sha256:<hash>","Size":<size>}} # when the push is successful, or # {"errorDetail": {"message":"<error_msg>"},"error":"<error_msg>"} # when the push is not successful last_line = json.loads(result.split("\n")[-1]) if "error" in last_line: self.client.images.remove(f"{self.use_registry_name}:{image_tag}") raise PushToRegistryError( f"Push of the image failed because of: {last_line['error']}", full_output=result) logger.info("Pushed image to registry %s:%s", self.use_registry_name, image_tag) logger.debug("Info:\n%s", result)
def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path) -> Result: """ Gets a path to a Python executable by calling the abstract method :meth:`get_image_for_repo <BaseRunInSubprocessBackend.get_image_for_repo>` and runs the task using :class:`subprocess.Popen` See :meth:`BaseBackend.run <arca.BaseBackend.run>` to see arguments description. """ python_path = self.get_or_create_environment(repo, branch, git_repo, repo_path) task_filename, task_json = self.serialized_task(task) task_path = Path(self._arca.base_dir, "tasks", task_filename) task_path.parent.mkdir(parents=True, exist_ok=True) task_path.write_text(task_json) logger.info("Stored task definition at %s", task_path) out_output = "" err_output = "" cwd = str(repo_path / self.cwd) logger.info("Running at cwd %s", cwd) try: logger.debug("Running with python %s", python_path) process = subprocess.Popen([python_path, str(self.RUNNER), str(task_path.resolve())], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) try: out_stream, err_stream = process.communicate(timeout=task.timeout) except subprocess.TimeoutExpired: process.kill() raise BuildTimeoutError(f"The task timeouted after {task.timeout} seconds.") out_output = out_stream.decode("utf-8") err_output = err_stream.decode("utf-8") logger.debug("stdout output from the command") logger.debug(out_output) return Result(out_output) except BuildError: # can be raised by :meth:`Result.__init__` or by timeout raise except Exception as e: logger.exception(e) raise BuildError("The build failed", extra_info={ "exception": e, "out_output": out_output, "err_output": err_output, })
def inject_arca(self, arca): """ Apart from the usual validation stuff it also creates log file for this instance. """ super().inject_arca(arca) import vagrant self.log_path = Path(self._arca.base_dir) / "logs" / (str(uuid4()) + ".log") self.log_path.parent.mkdir(exist_ok=True, parents=True) logger.info("Storing vagrant log in %s", self.log_path) self.log_cm = vagrant.make_file_cm(self.log_path)
def get_image_for_repo(self, repo: str, branch: str, git_repo: Repo, repo_path: Path): """ Returns an image for the specific repo (based on settings and requirements). 1. Checks if the image already exists locally 2. Tries to pull it from registry (if ``use_registry_name`` is set) 3. Builds the image 4. Pushes the image to registry so the image is available next time (if ``registry_pull_only`` is not set) See :meth:`run` for parameters descriptions. :rtype: docker.models.images.Image """ requirements_option, requirements_hash = self.get_requirements_information( repo_path) dependencies = self.get_dependencies() logger.info("Getting image based on %s, %s", requirements_option, dependencies) image_name = self.get_image_name(repo_path, requirements_option, dependencies) image_tag = self.get_image_tag(requirements_option, requirements_hash, dependencies) logger.info("Using image %s:%s", image_name, image_tag) if self.image_exists(image_name, image_tag): image = self.get_image(image_name, image_tag) # in case the push to registry was set later and the image wasn't pushed when built if self.use_registry_name is not None and not self.registry_pull_only: self.push_to_registry(image, image_tag) return image if self.use_registry_name is not None: # the target image might have been built and pushed in a previous run already, let's try to pull it image = self.try_pull_image_from_registry(image_name, image_tag) if image is not None: # image wasn't found return image image = self.build_image(image_name, image_tag, repo_path, requirements_option, dependencies) if self.use_registry_name is not None and not self.registry_pull_only: self.push_to_registry(image, image_tag) return image
def install_requirements(self, *, path: Optional[Path] = None, requirements: Optional[Iterable[str]] = None, _action: str = "install"): """ Installs requirements, either from a file or from a iterable of strings. :param path: :class:`Path <pathlib.Path>` to a ``requirements.txt`` file. Has priority over ``requirements``. :param requirements: A iterable of strings of requirements to install. :param _action: For testing purposes, can be either ``install`` or ``uninstall`` :raise BuildError: If installing fails. :raise ValueError: If both ``file`` and ``requirements`` are undefined. :raise ValueError: If ``_action`` not ``install`` or ``uninstall``. """ if _action not in ["install", "uninstall"]: raise ValueError(f"{_action} is invalid value for _action") cmd = [sys.executable, "-m", "pip", _action] if _action == "uninstall": cmd += ["-y"] if path is not None: cmd += ["-r", str(path)] elif requirements is not None: cmd += list(requirements) else: raise ValueError("Either path or requirements has to be provided") logger.info("Installing requirements with command: %s", cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: out_stream, err_stream = process.communicate(timeout=self.requirements_timeout) except subprocess.TimeoutExpired: process.kill() raise BuildTimeoutError(f"Installing of requirements timeouted after {self.requirements_timeout} seconds.") out_stream = out_stream.decode("utf-8") err_stream = err_stream.decode("utf-8") logger.debug("Return code is %s", process.returncode) logger.debug(out_stream) logger.debug(err_stream) if process.returncode: raise BuildError(f"Unable to {_action} requirements from the target repository", extra_info={ "out_stream": out_stream, "err_stream": err_stream, "returncode": process.returncode })
def handle_requirements(self, repo: str, branch: str, repo_path: Path): """ Checks the differences and handles it using the selected strategy. """ if self.requirements_strategy == RequirementsStrategy.IGNORE: logger.info("Requirements strategy is IGNORE") return requirements = repo_path / self.requirements_location # explicitly configured there are no requirements for the current environment if self.current_environment_requirements is None: if not requirements.exists(): return # no diff, since no requirements both in current env and repository requirements_set = self.get_requirements_set(requirements) if len(requirements_set): if self.requirements_strategy == RequirementsStrategy.RAISE: raise RequirementsMismatch(f"There are extra requirements in repository {repo}, branch {branch}.", diff=requirements.read_text()) self.install_requirements(path=requirements) # requirements for current environment configured else: current_requirements = Path(self.current_environment_requirements) if not requirements.exists(): return # no req. file in repo -> no extra requirements logger.info("Searching for current requirements at absolute path %s", current_requirements) if not current_requirements.exists(): raise ArcaMisconfigured("Can't locate current environment requirements.") current_requirements_set = self.get_requirements_set(current_requirements) requirements_set = self.get_requirements_set(requirements) # only requirements that are extra in repository requirements matter extra_requirements_set = requirements_set - current_requirements_set if len(extra_requirements_set) == 0: return # no extra requirements in repository else: if self.requirements_strategy == RequirementsStrategy.RAISE: raise RequirementsMismatch(f"There are extra requirements in repository {repo}, branch {branch}.", diff="\n".join(extra_requirements_set)) elif self.requirements_strategy == RequirementsStrategy.INSTALL_EXTRA: self.install_requirements(requirements=extra_requirements_set)
def try_pull_image_from_registry(self, image_name, image_tag): """ Tries to pull a image with the tag ``image_tag`` from registry set by ``use_registry_name``. After the image is pulled, it's tagged with ``image_name``:``image_tag`` so lookup can be made locally next time. :return: A :class:`Image <docker.models.images.Image>` instance if the image exists, ``None`` otherwise. :rtype: Optional[docker.models.images.Image] """ try: image = self.client.images.pull(self.use_registry_name, image_tag) except (docker.errors.ImageNotFound, docker.errors.NotFound): # the image doesn't exist logger.info("Tried to pull %s:%s from a registry, not found", self.use_registry_name, image_tag) return None logger.info("Pulled %s:%s from registry, tagged %s:%s", self.use_registry_name, image_tag, image_name, image_tag) # the name and tag are different on the repo, let's tag it with local name so exists checks run smoothly image.tag(image_name, image_tag) return image
def get_or_create_venv(self, path: Path) -> Path: """ Gets the location of the virtualenv from :meth:`get_virtualenv_path`, checks if it exists already, creates it and installs requirements otherwise. The virtualenvs are stored in a folder based on the :class:`Arca` ``base_dir`` setting. :param path: :class:`Path <pathlib.Path>` to the cloned repository. """ requirements_option, requirements_hash = self.get_requirements_information( path) venv_path = self.get_virtualenv_path(requirements_option, requirements_hash) if not venv_path.exists(): logger.info(f"Creating a venv in {venv_path}") builder = EnvBuilder(with_pip=True) builder.create(venv_path) shell = False cmd = None cwd = None if requirements_option == RequirementsOptions.pipfile: cmd = [ "source", (str(venv_path / "bin" / "activate")), "&&", "pipenv", "install", "--deploy", "--ignore-pipfile" ] cmd = " ".join(cmd) cwd = path / self.pipfile_location shell = True elif requirements_option == RequirementsOptions.requirements_txt: requirements_file = path / self.requirements_location logger.debug("Requirements file:") logger.debug(requirements_file.read_text()) logger.info("Installing requirements from %s", requirements_file) cmd = [ str(venv_path / "bin" / "python3"), "-m", "pip", "install", "-r", shlex.quote(str(requirements_file)) ] if cmd is not None: logger.info("Running Popen cmd %s, with shell %s", cmd, shell) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, cwd=cwd) try: out_stream, err_stream = process.communicate( timeout=self.requirements_timeout) except subprocess.TimeoutExpired: process.kill() logger.warning( "The install command timed out, deleting the virtualenv" ) shutil.rmtree(str(venv_path), ignore_errors=True) raise BuildTimeoutError( f"Installing of requirements timeouted after " f"{self.requirements_timeout} seconds.") out_stream = out_stream.decode("utf-8") err_stream = err_stream.decode("utf-8") logger.debug("Return code is %s", process.returncode) logger.debug(out_stream) logger.debug(err_stream) if process.returncode: logger.warning( "The install command failed, deleting the virtualenv") shutil.rmtree(str(venv_path), ignore_errors=True) raise BuildError("Unable to install requirements.txt", extra_info={ "out_stream": out_stream, "err_stream": err_stream, "returncode": process.returncode }) else: logger.info( "Requirements file not present in repo, empty venv it is.") else: logger.info(f"Venv already exists in {venv_path}") return venv_path
def get_or_build_image(self, name: str, tag: str, dockerfile: Union[str, Callable[..., str]], *, pull=True, build_context: Optional[Path] = None): """ A proxy for commonly built images, returns them from the local system if they exist, tries to pull them if pull isn't disabled, otherwise builds them by the definition in ``dockerfile``. :param name: Name of the image :param tag: Image tag :param dockerfile: Dockerfile text or a callable (no arguments) that produces Dockerfile text :param pull: If the image is not present locally, allow pulling from registry (default is ``True``) :param build_context: A path to a folder. If it's provided, docker will build the image in the context of this folder. (eg. if ``ADD`` is needed) """ if self.image_exists(name, tag): logger.info("Image %s:%s exists", name, tag) return elif pull: logger.info("Trying to pull image %s:%s", name, tag) try: self.client.images.pull(name, tag=tag) logger.info("The image %s:%s was pulled from registry", name, tag) return except docker.errors.APIError: logger.info( "The image %s:%s can't be pulled, building locally.", name, tag) if callable(dockerfile): dockerfile = dockerfile() try: if build_context is None: fileobj = BytesIO(bytes( dockerfile, "utf-8")) # required by the docker library self.client.images.build(fileobj=fileobj, tag=f"{name}:{tag}") else: dockerfile_file = build_context / "dockerfile" dockerfile_file.write_text(dockerfile) self.client.images.build(path=str(build_context.resolve()), dockerfile=dockerfile_file.name, tag=f"{name}:{tag}") dockerfile_file.unlink() except docker.errors.BuildError as e: for line in e.build_log: if isinstance(line, dict) and line.get( "errorDetail") and line["errorDetail"].get("code") in { 124, 143 }: raise BuildTimeoutError( f"Installing of requirements timeouted after " f"{self.requirements_timeout} seconds.") logger.exception(e) raise BuildError( "Building docker image failed, see extra info for details.", extra_info={"build_log": e.build_log})
def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path): """ Starts up a VM, builds an docker image and gets it to the VM, runs the script over SSH, returns result. Stops the VM if ``keep_vm_running`` is not set. """ from fabric import api from fabric.exceptions import CommandTimeout # start up or get running VM vm_location = self.get_vm_location() self.ensure_vm_running(vm_location) logger.info("Running with VM located at %s", vm_location) # pushes the image to the registry so it can be pulled in the VM self.check_docker_access() # init client self.get_image_for_repo(repo, branch, git_repo, repo_path) # getting things needed for execution over SSH image_tag = self.get_image_tag(self.get_requirements_file(repo_path), self.get_dependencies()) image_name = self.use_registry_name task_filename, task_json = self.serialized_task(task) (vm_location / task_filename).write_text(task_json) container_name = self.get_container_name(repo, branch, git_repo) # setting up Fabric api.env.hosts = [self.vagrant.user_hostname_port()] api.env.key_filename = self.vagrant.keyfile() api.env.disable_known_hosts = True # useful for when the vagrant box ip changes. api.env.abort_exception = BuildError # raises SystemExit otherwise api.env.shell = "/bin/sh -l -c" if self.quiet: api.output.everything = False else: api.output.everything = True # executes the task try: res = api.execute(self.fabric_task, container_name=container_name, definition_filename=task_filename, image_name=image_name, image_tag=image_tag, repository=str(repo_path.relative_to(Path(self._arca.base_dir).resolve() / 'repos')), timeout=task.timeout) return Result(res[self.vagrant.user_hostname_port()].stdout) except CommandTimeout: raise BuildTimeoutError(f"The task timeouted after {task.timeout} seconds.") except BuildError: # can be raised by :meth:`Result.__init__` raise except Exception as e: logger.exception(e) raise BuildError("The build failed", extra_info={ "exception": e }) finally: # stops or destroys the VM if it should not be kept running if not self.keep_vm_running: if self.destroy: self.vagrant.destroy() shutil.rmtree(self.vagrant.root, ignore_errors=True) self.vagrant = None else: self.vagrant.halt()
def get_or_create_venv(self, path: Path) -> Path: """ Gets the name of the virtualenv from :meth:`get_virtualenv_name`, checks if it exists already, creates it and installs requirements otherwise. The virtualenvs are stored in a folder based on the :class:`Arca` ``base_dir`` setting. :param path: :class:`Path <pathlib.Path>` to the cloned repository. """ requirements_file = self.get_requirements_file(path) venv_name = self.get_virtualenv_name(requirements_file) venv_path = Path(self._arca.base_dir) / "venvs" / venv_name if not venv_path.exists(): logger.info(f"Creating a venv in {venv_path}") builder = EnvBuilder(with_pip=True) builder.create(venv_path) if requirements_file is not None: logger.debug("Requirements file:") logger.debug(requirements_file.read_text()) logger.info("Installing requirements from %s", requirements_file) process = subprocess.Popen([ str(venv_path / "bin" / "python3"), "-m", "pip", "install", "-r", str(requirements_file) ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: out_stream, err_stream = process.communicate( timeout=self.requirements_timeout) except subprocess.TimeoutExpired: process.kill() shutil.rmtree(venv_path, ignore_errors=True) raise BuildTimeoutError( f"Installing of requirements timeouted after " f"{self.requirements_timeout} seconds.") out_stream = out_stream.decode("utf-8") err_stream = err_stream.decode("utf-8") logger.debug("Return code is %s", process.returncode) logger.debug(out_stream) logger.debug(err_stream) if process.returncode: venv_path.rmdir() raise BuildError("Unable to install requirements.txt", extra_info={ "out_stream": out_stream, "err_stream": err_stream, "returncode": process.returncode }) else: logger.info( "Requirements file not present in repo, empty venv it is.") else: logger.info(f"Venv already eixsts in {venv_path}") return venv_path