def _pip_action(action, package):
if action not in ["install", "uninstall"]:
raise ValueError(f"{action} is invalid value for action")
cmd = [sys.executable, "-m", "pip", action]
if action == "uninstall":
cmd += ["-y"]
cmd += [package]
logger.info("Installing requirements with command: %s", cmd)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out_stream, err_stream = process.communicate()
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
def init_vagrant(self, vagrant_file):
""" Creates a Vagrantfile in the target dir, with only the base image pulled.
Copies the runner script to the directory so it's accessible from the VM.
"""
if self.inherit_image:
image_name, image_tag = str(self.inherit_image).split(":")
else:
image_name = self.get_arca_base_name()
image_tag = self.get_python_base_tag(self.get_python_version())
logger.info("Creating Vagrantfile located in %s, base image %s:%s", vagrant_file, image_name, image_tag)
repos_dir = (Path(self._arca.base_dir) / 'repos').resolve()
vagrant_file.parent.mkdir(exist_ok=True, parents=True)
vagrant_file.write_text(dedent(f"""
# -*- mode: ruby -*-
# vi: set ft=ruby :
Vagrant.configure("2") do |config|
config.vm.box = "{self.box}"
config.ssh.insert_key = true
config.vm.provision "docker" do |d|
d.pull_images "{image_name}:{image_tag}"
end
config.vm.synced_folder ".", "/vagrant"
config.vm.synced_folder "{repos_dir}", "/srv/repos"
config.vm.provider "{self.provider}"
end
"""))
(vagrant_file.parent / "runner.py").write_text(self.RUNNER.read_text())
def push_to_registry(self, image, image_tag: str):
""" Pushes a local image to a registry based on the ``use_registry_name`` setting.
:type image: docker.models.images.Image
:raise PushToRegistryError: If the push fails.
"""
# already tagged, so it's already pushed
if f"{self.use_registry_name}:{image_tag}" in image.tags:
return
image.tag(self.use_registry_name, image_tag)
result = self.client.images.push(self.use_registry_name, image_tag)
result = result.strip() # remove empty line at the end of output
# the last can have one of two outputs, either
# {"progressDetail":{},"aux":{"Tag":"<tag>","Digest":"sha256:<hash>","Size":<size>}}
# when the push is successful, or
# {"errorDetail": {"message":"<error_msg>"},"error":"<error_msg>"}
# when the push is not successful
last_line = json.loads(result.split("\n")[-1])
if "error" in last_line:
self.client.images.remove(f"{self.use_registry_name}:{image_tag}")
raise PushToRegistryError(
f"Push of the image failed because of: {last_line['error']}",
full_output=result)
logger.info("Pushed image to registry %s:%s", self.use_registry_name,
image_tag)
logger.debug("Info:\n%s", result)
def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path) -> Result:
"""
Gets a path to a Python executable by calling the abstract method
:meth:`get_image_for_repo <BaseRunInSubprocessBackend.get_image_for_repo>`
and runs the task using :class:`subprocess.Popen`
See :meth:`BaseBackend.run <arca.BaseBackend.run>` to see arguments description.
"""
python_path = self.get_or_create_environment(repo, branch, git_repo, repo_path)
task_filename, task_json = self.serialized_task(task)
task_path = Path(self._arca.base_dir, "tasks", task_filename)
task_path.parent.mkdir(parents=True, exist_ok=True)
task_path.write_text(task_json)
logger.info("Stored task definition at %s", task_path)
out_output = ""
err_output = ""
cwd = str(repo_path / self.cwd)
logger.info("Running at cwd %s", cwd)
try:
logger.debug("Running with python %s", python_path)
process = subprocess.Popen([python_path,
str(self.RUNNER),
str(task_path.resolve())],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
try:
out_stream, err_stream = process.communicate(timeout=task.timeout)
except subprocess.TimeoutExpired:
process.kill()
raise BuildTimeoutError(f"The task timeouted after {task.timeout} seconds.")
out_output = out_stream.decode("utf-8")
err_output = err_stream.decode("utf-8")
logger.debug("stdout output from the command")
logger.debug(out_output)
return Result(out_output)
except BuildError: # can be raised by :meth:`Result.__init__` or by timeout
raise
except Exception as e:
logger.exception(e)
raise BuildError("The build failed", extra_info={
"exception": e,
"out_output": out_output,
"err_output": err_output,
})
def inject_arca(self, arca):
""" Apart from the usual validation stuff it also creates log file for this instance.
"""
super().inject_arca(arca)
import vagrant
self.log_path = Path(self._arca.base_dir) / "logs" / (str(uuid4()) + ".log")
self.log_path.parent.mkdir(exist_ok=True, parents=True)
logger.info("Storing vagrant log in %s", self.log_path)
self.log_cm = vagrant.make_file_cm(self.log_path)
def get_image_for_repo(self, repo: str, branch: str, git_repo: Repo,
repo_path: Path):
"""
Returns an image for the specific repo (based on settings and requirements).
1. Checks if the image already exists locally
2. Tries to pull it from registry (if ``use_registry_name`` is set)
3. Builds the image
4. Pushes the image to registry so the image is available next time (if ``registry_pull_only`` is not set)
See :meth:`run` for parameters descriptions.
:rtype: docker.models.images.Image
"""
requirements_option, requirements_hash = self.get_requirements_information(
repo_path)
dependencies = self.get_dependencies()
logger.info("Getting image based on %s, %s", requirements_option,
dependencies)
image_name = self.get_image_name(repo_path, requirements_option,
dependencies)
image_tag = self.get_image_tag(requirements_option, requirements_hash,
dependencies)
logger.info("Using image %s:%s", image_name, image_tag)
if self.image_exists(image_name, image_tag):
image = self.get_image(image_name, image_tag)
# in case the push to registry was set later and the image wasn't pushed when built
if self.use_registry_name is not None and not self.registry_pull_only:
self.push_to_registry(image, image_tag)
return image
if self.use_registry_name is not None:
# the target image might have been built and pushed in a previous run already, let's try to pull it
image = self.try_pull_image_from_registry(image_name, image_tag)
if image is not None: # image wasn't found
return image
image = self.build_image(image_name, image_tag, repo_path,
requirements_option, dependencies)
if self.use_registry_name is not None and not self.registry_pull_only:
self.push_to_registry(image, image_tag)
return image
def install_requirements(self, *, path: Optional[Path] = None, requirements: Optional[Iterable[str]] = None,
_action: str = "install"):
"""
Installs requirements, either from a file or from a iterable of strings.
:param path: :class:`Path <pathlib.Path>` to a ``requirements.txt`` file. Has priority over ``requirements``.
:param requirements: A iterable of strings of requirements to install.
:param _action: For testing purposes, can be either ``install`` or ``uninstall``
:raise BuildError: If installing fails.
:raise ValueError: If both ``file`` and ``requirements`` are undefined.
:raise ValueError: If ``_action`` not ``install`` or ``uninstall``.
"""
if _action not in ["install", "uninstall"]:
raise ValueError(f"{_action} is invalid value for _action")
cmd = [sys.executable, "-m", "pip", _action]
if _action == "uninstall":
cmd += ["-y"]
if path is not None:
cmd += ["-r", str(path)]
elif requirements is not None:
cmd += list(requirements)
else:
raise ValueError("Either path or requirements has to be provided")
logger.info("Installing requirements with command: %s", cmd)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
out_stream, err_stream = process.communicate(timeout=self.requirements_timeout)
except subprocess.TimeoutExpired:
process.kill()
raise BuildTimeoutError(f"Installing of requirements timeouted after {self.requirements_timeout} seconds.")
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
if process.returncode:
raise BuildError(f"Unable to {_action} requirements from the target repository", extra_info={
"out_stream": out_stream,
"err_stream": err_stream,
"returncode": process.returncode
})
def handle_requirements(self, repo: str, branch: str, repo_path: Path):
""" Checks the differences and handles it using the selected strategy.
"""
if self.requirements_strategy == RequirementsStrategy.IGNORE:
logger.info("Requirements strategy is IGNORE")
return
requirements = repo_path / self.requirements_location
# explicitly configured there are no requirements for the current environment
if self.current_environment_requirements is None:
if not requirements.exists():
return # no diff, since no requirements both in current env and repository
requirements_set = self.get_requirements_set(requirements)
if len(requirements_set):
if self.requirements_strategy == RequirementsStrategy.RAISE:
raise RequirementsMismatch(f"There are extra requirements in repository {repo}, branch {branch}.",
diff=requirements.read_text())
self.install_requirements(path=requirements)
# requirements for current environment configured
else:
current_requirements = Path(self.current_environment_requirements)
if not requirements.exists():
return # no req. file in repo -> no extra requirements
logger.info("Searching for current requirements at absolute path %s", current_requirements)
if not current_requirements.exists():
raise ArcaMisconfigured("Can't locate current environment requirements.")
current_requirements_set = self.get_requirements_set(current_requirements)
requirements_set = self.get_requirements_set(requirements)
# only requirements that are extra in repository requirements matter
extra_requirements_set = requirements_set - current_requirements_set
if len(extra_requirements_set) == 0:
return # no extra requirements in repository
else:
if self.requirements_strategy == RequirementsStrategy.RAISE:
raise RequirementsMismatch(f"There are extra requirements in repository {repo}, branch {branch}.",
diff="\n".join(extra_requirements_set))
elif self.requirements_strategy == RequirementsStrategy.INSTALL_EXTRA:
self.install_requirements(requirements=extra_requirements_set)
def try_pull_image_from_registry(self, image_name, image_tag):
"""
Tries to pull a image with the tag ``image_tag`` from registry set by ``use_registry_name``.
After the image is pulled, it's tagged with ``image_name``:``image_tag`` so lookup can
be made locally next time.
:return: A :class:`Image <docker.models.images.Image>` instance if the image exists, ``None`` otherwise.
:rtype: Optional[docker.models.images.Image]
"""
try:
image = self.client.images.pull(self.use_registry_name, image_tag)
except (docker.errors.ImageNotFound,
docker.errors.NotFound): # the image doesn't exist
logger.info("Tried to pull %s:%s from a registry, not found",
self.use_registry_name, image_tag)
return None
logger.info("Pulled %s:%s from registry, tagged %s:%s",
self.use_registry_name, image_tag, image_name, image_tag)
# the name and tag are different on the repo, let's tag it with local name so exists checks run smoothly
image.tag(image_name, image_tag)
return image
def get_or_create_venv(self, path: Path) -> Path:
"""
Gets the location of the virtualenv from :meth:`get_virtualenv_path`, checks if it exists already,
creates it and installs requirements otherwise. The virtualenvs are stored in a folder based
on the :class:`Arca` ``base_dir`` setting.
:param path: :class:`Path <pathlib.Path>` to the cloned repository.
"""
requirements_option, requirements_hash = self.get_requirements_information(
path)
venv_path = self.get_virtualenv_path(requirements_option,
requirements_hash)
if not venv_path.exists():
logger.info(f"Creating a venv in {venv_path}")
builder = EnvBuilder(with_pip=True)
builder.create(venv_path)
shell = False
cmd = None
cwd = None
if requirements_option == RequirementsOptions.pipfile:
cmd = [
"source", (str(venv_path / "bin" / "activate")), "&&",
"pipenv", "install", "--deploy", "--ignore-pipfile"
]
cmd = " ".join(cmd)
cwd = path / self.pipfile_location
shell = True
elif requirements_option == RequirementsOptions.requirements_txt:
requirements_file = path / self.requirements_location
logger.debug("Requirements file:")
logger.debug(requirements_file.read_text())
logger.info("Installing requirements from %s",
requirements_file)
cmd = [
str(venv_path / "bin" / "python3"), "-m", "pip", "install",
"-r",
shlex.quote(str(requirements_file))
]
if cmd is not None:
logger.info("Running Popen cmd %s, with shell %s", cmd, shell)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=shell,
cwd=cwd)
try:
out_stream, err_stream = process.communicate(
timeout=self.requirements_timeout)
except subprocess.TimeoutExpired:
process.kill()
logger.warning(
"The install command timed out, deleting the virtualenv"
)
shutil.rmtree(str(venv_path), ignore_errors=True)
raise BuildTimeoutError(
f"Installing of requirements timeouted after "
f"{self.requirements_timeout} seconds.")
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
if process.returncode:
logger.warning(
"The install command failed, deleting the virtualenv")
shutil.rmtree(str(venv_path), ignore_errors=True)
raise BuildError("Unable to install requirements.txt",
extra_info={
"out_stream": out_stream,
"err_stream": err_stream,
"returncode": process.returncode
})
else:
logger.info(
"Requirements file not present in repo, empty venv it is.")
else:
logger.info(f"Venv already exists in {venv_path}")
return venv_path
def get_or_build_image(self,
name: str,
tag: str,
dockerfile: Union[str, Callable[..., str]],
*,
pull=True,
build_context: Optional[Path] = None):
"""
A proxy for commonly built images, returns them from the local system if they exist, tries to pull them if
pull isn't disabled, otherwise builds them by the definition in ``dockerfile``.
:param name: Name of the image
:param tag: Image tag
:param dockerfile: Dockerfile text or a callable (no arguments) that produces Dockerfile text
:param pull: If the image is not present locally, allow pulling from registry (default is ``True``)
:param build_context: A path to a folder. If it's provided, docker will build the image in the context
of this folder. (eg. if ``ADD`` is needed)
"""
if self.image_exists(name, tag):
logger.info("Image %s:%s exists", name, tag)
return
elif pull:
logger.info("Trying to pull image %s:%s", name, tag)
try:
self.client.images.pull(name, tag=tag)
logger.info("The image %s:%s was pulled from registry", name,
tag)
return
except docker.errors.APIError:
logger.info(
"The image %s:%s can't be pulled, building locally.", name,
tag)
if callable(dockerfile):
dockerfile = dockerfile()
try:
if build_context is None:
fileobj = BytesIO(bytes(
dockerfile, "utf-8")) # required by the docker library
self.client.images.build(fileobj=fileobj, tag=f"{name}:{tag}")
else:
dockerfile_file = build_context / "dockerfile"
dockerfile_file.write_text(dockerfile)
self.client.images.build(path=str(build_context.resolve()),
dockerfile=dockerfile_file.name,
tag=f"{name}:{tag}")
dockerfile_file.unlink()
except docker.errors.BuildError as e:
for line in e.build_log:
if isinstance(line, dict) and line.get(
"errorDetail") and line["errorDetail"].get("code") in {
124, 143
}:
raise BuildTimeoutError(
f"Installing of requirements timeouted after "
f"{self.requirements_timeout} seconds.")
logger.exception(e)
raise BuildError(
"Building docker image failed, see extra info for details.",
extra_info={"build_log": e.build_log})
def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path):
""" Starts up a VM, builds an docker image and gets it to the VM, runs the script over SSH, returns result.
Stops the VM if ``keep_vm_running`` is not set.
"""
from fabric import api
from fabric.exceptions import CommandTimeout
# start up or get running VM
vm_location = self.get_vm_location()
self.ensure_vm_running(vm_location)
logger.info("Running with VM located at %s", vm_location)
# pushes the image to the registry so it can be pulled in the VM
self.check_docker_access() # init client
self.get_image_for_repo(repo, branch, git_repo, repo_path)
# getting things needed for execution over SSH
image_tag = self.get_image_tag(self.get_requirements_file(repo_path), self.get_dependencies())
image_name = self.use_registry_name
task_filename, task_json = self.serialized_task(task)
(vm_location / task_filename).write_text(task_json)
container_name = self.get_container_name(repo, branch, git_repo)
# setting up Fabric
api.env.hosts = [self.vagrant.user_hostname_port()]
api.env.key_filename = self.vagrant.keyfile()
api.env.disable_known_hosts = True # useful for when the vagrant box ip changes.
api.env.abort_exception = BuildError # raises SystemExit otherwise
api.env.shell = "/bin/sh -l -c"
if self.quiet:
api.output.everything = False
else:
api.output.everything = True
# executes the task
try:
res = api.execute(self.fabric_task,
container_name=container_name,
definition_filename=task_filename,
image_name=image_name,
image_tag=image_tag,
repository=str(repo_path.relative_to(Path(self._arca.base_dir).resolve() / 'repos')),
timeout=task.timeout)
return Result(res[self.vagrant.user_hostname_port()].stdout)
except CommandTimeout:
raise BuildTimeoutError(f"The task timeouted after {task.timeout} seconds.")
except BuildError: # can be raised by :meth:`Result.__init__`
raise
except Exception as e:
logger.exception(e)
raise BuildError("The build failed", extra_info={
"exception": e
})
finally:
# stops or destroys the VM if it should not be kept running
if not self.keep_vm_running:
if self.destroy:
self.vagrant.destroy()
shutil.rmtree(self.vagrant.root, ignore_errors=True)
self.vagrant = None
else:
self.vagrant.halt()
def get_or_create_venv(self, path: Path) -> Path:
"""
Gets the name of the virtualenv from :meth:`get_virtualenv_name`, checks if it exists already,
creates it and installs requirements otherwise. The virtualenvs are stored in a folder based
on the :class:`Arca` ``base_dir`` setting.
:param path: :class:`Path <pathlib.Path>` to the cloned repository.
"""
requirements_file = self.get_requirements_file(path)
venv_name = self.get_virtualenv_name(requirements_file)
venv_path = Path(self._arca.base_dir) / "venvs" / venv_name
if not venv_path.exists():
logger.info(f"Creating a venv in {venv_path}")
builder = EnvBuilder(with_pip=True)
builder.create(venv_path)
if requirements_file is not None:
logger.debug("Requirements file:")
logger.debug(requirements_file.read_text())
logger.info("Installing requirements from %s",
requirements_file)
process = subprocess.Popen([
str(venv_path / "bin" / "python3"), "-m", "pip", "install",
"-r",
str(requirements_file)
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
out_stream, err_stream = process.communicate(
timeout=self.requirements_timeout)
except subprocess.TimeoutExpired:
process.kill()
shutil.rmtree(venv_path, ignore_errors=True)
raise BuildTimeoutError(
f"Installing of requirements timeouted after "
f"{self.requirements_timeout} seconds.")
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
if process.returncode:
venv_path.rmdir()
raise BuildError("Unable to install requirements.txt",
extra_info={
"out_stream": out_stream,
"err_stream": err_stream,
"returncode": process.returncode
})
else:
logger.info(
"Requirements file not present in repo, empty venv it is.")
else:
logger.info(f"Venv already eixsts in {venv_path}")
return venv_path