def get_requirements_information(self, path: Path) -> Tuple[RequirementsOptions, Optional[str]]:
"""
Returns the information needed to install requirements for a repository - what kind is used and the hash
of contents of the defining file.
"""
if self.pipfile_location is not None:
pipfile = path / self.pipfile_location / "Pipfile"
pipfile_lock = path / self.pipfile_location / "Pipfile.lock"
pipfile_exists = pipfile.exists()
pipfile_lock_exists = pipfile_lock.exists()
if pipfile_exists and pipfile_lock_exists:
option = RequirementsOptions.pipfile
return option, self.hash_file_contents(option, pipfile_lock)
elif pipfile_exists:
raise BuildError("Only the Pipfile is included in the repository, Arca does not support that.")
elif pipfile_lock_exists:
raise BuildError("Only the Pipfile.lock file is include in the repository, Arca does not support that.")
if self.requirements_location:
requirements_file = path / self.requirements_location
if requirements_file.exists():
option = RequirementsOptions.requirements_txt
return option, self.hash_file_contents(option, requirements_file)
return RequirementsOptions.no_requirements, None
def __init__(self, result: Union[str, Dict[str, Any]]) -> None:
if isinstance(result, (str, bytes, bytearray)):
output = result
try:
result = json.loads(result)
except ValueError:
raise BuildError(
"The build failed (the output was corrupted, "
"possibly by the callable printing something)",
extra_info={"output": output})
if not isinstance(result, dict):
raise BuildError(
"The build failed (the value returned from the runner was not valid)"
)
if not result.get("success"):
reason = "Task failed"
if result.get("reason") == "corrupted_definition":
reason = "Task failed because the definition was corrupted."
elif result.get("reason") == "import":
reason = "Task failed beucase the entry point could not be imported"
raise BuildError(reason,
extra_info={"traceback": result.get("error")})
#: The output of the task
self.output = result.get("result")
def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path) -> Result:
"""
Gets a path to a Python executable by calling the abstract method
:meth:`get_image_for_repo <BaseRunInSubprocessBackend.get_image_for_repo>`
and runs the task using :class:`subprocess.Popen`
See :meth:`BaseBackend.run <arca.BaseBackend.run>` to see arguments description.
"""
python_path = self.get_or_create_environment(repo, branch, git_repo, repo_path)
task_filename, task_json = self.serialized_task(task)
task_path = Path(self._arca.base_dir, "tasks", task_filename)
task_path.parent.mkdir(parents=True, exist_ok=True)
task_path.write_text(task_json)
logger.info("Stored task definition at %s", task_path)
out_output = ""
err_output = ""
cwd = str(repo_path / self.cwd)
logger.info("Running at cwd %s", cwd)
try:
logger.debug("Running with python %s", python_path)
process = subprocess.Popen([python_path,
str(self.RUNNER),
str(task_path.resolve())],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
try:
out_stream, err_stream = process.communicate(timeout=task.timeout)
except subprocess.TimeoutExpired:
process.kill()
raise BuildTimeoutError(f"The task timeouted after {task.timeout} seconds.")
out_output = out_stream.decode("utf-8")
err_output = err_stream.decode("utf-8")
logger.debug("stdout output from the command")
logger.debug(out_output)
return Result(out_output)
except BuildError: # can be raised by :meth:`Result.__init__` or by timeout
raise
except Exception as e:
logger.exception(e)
raise BuildError("The build failed", extra_info={
"exception": e,
"out_output": out_output,
"err_output": err_output,
})
def install_requirements(self, *, path: Optional[Path] = None, requirements: Optional[Iterable[str]] = None,
_action: str = "install"):
"""
Installs requirements, either from a file or from a iterable of strings.
:param path: :class:`Path <pathlib.Path>` to a ``requirements.txt`` file. Has priority over ``requirements``.
:param requirements: A iterable of strings of requirements to install.
:param _action: For testing purposes, can be either ``install`` or ``uninstall``
:raise BuildError: If installing fails.
:raise ValueError: If both ``file`` and ``requirements`` are undefined.
:raise ValueError: If ``_action`` not ``install`` or ``uninstall``.
"""
if _action not in ["install", "uninstall"]:
raise ValueError(f"{_action} is invalid value for _action")
cmd = [sys.executable, "-m", "pip", _action]
if _action == "uninstall":
cmd += ["-y"]
if path is not None:
cmd += ["-r", str(path)]
elif requirements is not None:
cmd += list(requirements)
else:
raise ValueError("Either path or requirements has to be provided")
logger.info("Installing requirements with command: %s", cmd)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
out_stream, err_stream = process.communicate(timeout=self.requirements_timeout)
except subprocess.TimeoutExpired:
process.kill()
raise BuildTimeoutError(f"Installing of requirements timeouted after {self.requirements_timeout} seconds.")
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
if process.returncode:
raise BuildError(f"Unable to {_action} requirements from the target repository", extra_info={
"out_stream": out_stream,
"err_stream": err_stream,
"returncode": process.returncode
})
def check_docker_access(self):
""" Creates a :class:`DockerClient <docker.client.DockerClient>` for the instance and checks the connection.
:raise BuildError: If docker isn't accessible by the current user.
"""
try:
if self.client is None:
self.client = docker.from_env()
self.client.ping(
) # check that docker is running and user is permitted to access it
except ConnectionError as e:
logger.exception(e)
raise BuildError(
"Docker is not running or the current user doesn't have permissions to access docker."
)
def ensure_vm_running(self, vm_location):
""" Gets or creates a Vagrantfile in ``vm_location`` and calls ``vagrant up`` if the VM is not running.
"""
import vagrant
if self.vagrant is None:
vagrant_file = vm_location / "Vagrantfile"
if not vagrant_file.exists():
self.init_vagrant(vagrant_file)
self.vagrant = vagrant.Vagrant(vm_location,
quiet_stdout=self.quiet,
quiet_stderr=self.quiet,
out_cm=self.log_cm,
err_cm=self.log_cm)
status = [x for x in self.vagrant.status() if x.name == "default"][0]
if status.state != "running":
try:
self.vagrant.up()
except subprocess.CalledProcessError:
raise BuildError("Vagrant VM couldn't up launched. See output for details.")
def get_or_create_venv(self, path: Path) -> Path:
"""
Gets the location of the virtualenv from :meth:`get_virtualenv_path`, checks if it exists already,
creates it and installs requirements otherwise. The virtualenvs are stored in a folder based
on the :class:`Arca` ``base_dir`` setting.
:param path: :class:`Path <pathlib.Path>` to the cloned repository.
"""
requirements_option, requirements_hash = self.get_requirements_information(
path)
venv_path = self.get_virtualenv_path(requirements_option,
requirements_hash)
if not venv_path.exists():
logger.info(f"Creating a venv in {venv_path}")
builder = EnvBuilder(with_pip=True)
builder.create(venv_path)
shell = False
cmd = None
cwd = None
if requirements_option == RequirementsOptions.pipfile:
cmd = [
"source", (str(venv_path / "bin" / "activate")), "&&",
"pipenv", "install", "--deploy", "--ignore-pipfile"
]
cmd = " ".join(cmd)
cwd = path / self.pipfile_location
shell = True
elif requirements_option == RequirementsOptions.requirements_txt:
requirements_file = path / self.requirements_location
logger.debug("Requirements file:")
logger.debug(requirements_file.read_text())
logger.info("Installing requirements from %s",
requirements_file)
cmd = [
str(venv_path / "bin" / "python3"), "-m", "pip", "install",
"-r",
shlex.quote(str(requirements_file))
]
if cmd is not None:
logger.info("Running Popen cmd %s, with shell %s", cmd, shell)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=shell,
cwd=cwd)
try:
out_stream, err_stream = process.communicate(
timeout=self.requirements_timeout)
except subprocess.TimeoutExpired:
process.kill()
logger.warning(
"The install command timed out, deleting the virtualenv"
)
shutil.rmtree(str(venv_path), ignore_errors=True)
raise BuildTimeoutError(
f"Installing of requirements timeouted after "
f"{self.requirements_timeout} seconds.")
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
if process.returncode:
logger.warning(
"The install command failed, deleting the virtualenv")
shutil.rmtree(str(venv_path), ignore_errors=True)
raise BuildError("Unable to install requirements.txt",
extra_info={
"out_stream": out_stream,
"err_stream": err_stream,
"returncode": process.returncode
})
else:
logger.info(
"Requirements file not present in repo, empty venv it is.")
else:
logger.info(f"Venv already exists in {venv_path}")
return venv_path
def run(self, repo: str, branch: str, task: Task, git_repo: Repo,
repo_path: Path) -> Result:
""" Gets or builds an image for the repo, gets or starts a container for the image and runs the script.
:param repo: Repository URL
:param branch: Branch ane
:param task: :class:`Task` to run.
:param git_repo: :class:`Repo <git.repo.base.Repo>` of the cloned repository.
:param repo_path: :class:`Path <pathlib.Path>` to the cloned location.
"""
self.check_docker_access()
container_name = self.get_container_name(repo, branch, git_repo)
container = self.container_running(container_name)
if container is None:
image = self.get_image_for_repo(repo, branch, git_repo, repo_path)
container = self.start_container(image, container_name, repo_path)
task_filename, task_json = self.serialized_task(task)
container.put_archive(
"/srv/scripts", self.tar_task_definition(task_filename, task_json))
res = None
try:
command = ["timeout"]
if self.inherit_image:
if self.alpine_inherited or b"Alpine" in container.exec_run(
["cat", "/etc/issue"], tty=True).output:
self.alpine_inherited = True
command = ["timeout", "-t"]
command += [
str(task.timeout), "python", "/srv/scripts/runner.py",
f"/srv/scripts/{task_filename}"
]
logger.debug("Running command %s", " ".join(command))
res = container.exec_run(command, tty=True)
# 124 is the standard, 143 on alpine
if res.exit_code in {124, 143}:
raise BuildTimeoutError(
f"The task timeouted after {task.timeout} seconds.")
return Result(res.output)
except BuildError: # can be raised by :meth:`Result.__init__`
raise
except Exception as e:
logger.exception(e)
if res is not None:
logger.warning(res.output)
raise BuildError("The build failed",
extra_info={
"exception": e,
"output": res if res is None else res.output
})
finally:
if not self.keep_container_running:
container.kill(signal.SIGKILL)
else:
self._containers.add(container)
def get_or_build_image(self,
name: str,
tag: str,
dockerfile: Union[str, Callable[..., str]],
*,
pull=True,
build_context: Optional[Path] = None):
"""
A proxy for commonly built images, returns them from the local system if they exist, tries to pull them if
pull isn't disabled, otherwise builds them by the definition in ``dockerfile``.
:param name: Name of the image
:param tag: Image tag
:param dockerfile: Dockerfile text or a callable (no arguments) that produces Dockerfile text
:param pull: If the image is not present locally, allow pulling from registry (default is ``True``)
:param build_context: A path to a folder. If it's provided, docker will build the image in the context
of this folder. (eg. if ``ADD`` is needed)
"""
if self.image_exists(name, tag):
logger.info("Image %s:%s exists", name, tag)
return
elif pull:
logger.info("Trying to pull image %s:%s", name, tag)
try:
self.client.images.pull(name, tag=tag)
logger.info("The image %s:%s was pulled from registry", name,
tag)
return
except docker.errors.APIError:
logger.info(
"The image %s:%s can't be pulled, building locally.", name,
tag)
if callable(dockerfile):
dockerfile = dockerfile()
try:
if build_context is None:
fileobj = BytesIO(bytes(
dockerfile, "utf-8")) # required by the docker library
self.client.images.build(fileobj=fileobj, tag=f"{name}:{tag}")
else:
dockerfile_file = build_context / "dockerfile"
dockerfile_file.write_text(dockerfile)
self.client.images.build(path=str(build_context.resolve()),
dockerfile=dockerfile_file.name,
tag=f"{name}:{tag}")
dockerfile_file.unlink()
except docker.errors.BuildError as e:
for line in e.build_log:
if isinstance(line, dict) and line.get(
"errorDetail") and line["errorDetail"].get("code") in {
124, 143
}:
raise BuildTimeoutError(
f"Installing of requirements timeouted after "
f"{self.requirements_timeout} seconds.")
logger.exception(e)
raise BuildError(
"Building docker image failed, see extra info for details.",
extra_info={"build_log": e.build_log})
def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path):
""" Starts up a VM, builds an docker image and gets it to the VM, runs the script over SSH, returns result.
Stops the VM if ``keep_vm_running`` is not set.
"""
from fabric import api
from fabric.exceptions import CommandTimeout
# start up or get running VM
vm_location = self.get_vm_location()
self.ensure_vm_running(vm_location)
logger.info("Running with VM located at %s", vm_location)
# pushes the image to the registry so it can be pulled in the VM
self.check_docker_access() # init client
self.get_image_for_repo(repo, branch, git_repo, repo_path)
# getting things needed for execution over SSH
image_tag = self.get_image_tag(self.get_requirements_file(repo_path), self.get_dependencies())
image_name = self.use_registry_name
task_filename, task_json = self.serialized_task(task)
(vm_location / task_filename).write_text(task_json)
container_name = self.get_container_name(repo, branch, git_repo)
# setting up Fabric
api.env.hosts = [self.vagrant.user_hostname_port()]
api.env.key_filename = self.vagrant.keyfile()
api.env.disable_known_hosts = True # useful for when the vagrant box ip changes.
api.env.abort_exception = BuildError # raises SystemExit otherwise
api.env.shell = "/bin/sh -l -c"
if self.quiet:
api.output.everything = False
else:
api.output.everything = True
# executes the task
try:
res = api.execute(self.fabric_task,
container_name=container_name,
definition_filename=task_filename,
image_name=image_name,
image_tag=image_tag,
repository=str(repo_path.relative_to(Path(self._arca.base_dir).resolve() / 'repos')),
timeout=task.timeout)
return Result(res[self.vagrant.user_hostname_port()].stdout)
except CommandTimeout:
raise BuildTimeoutError(f"The task timeouted after {task.timeout} seconds.")
except BuildError: # can be raised by :meth:`Result.__init__`
raise
except Exception as e:
logger.exception(e)
raise BuildError("The build failed", extra_info={
"exception": e
})
finally:
# stops or destroys the VM if it should not be kept running
if not self.keep_vm_running:
if self.destroy:
self.vagrant.destroy()
shutil.rmtree(self.vagrant.root, ignore_errors=True)
self.vagrant = None
else:
self.vagrant.halt()
def get_or_create_venv(self, path: Path) -> Path:
"""
Gets the name of the virtualenv from :meth:`get_virtualenv_name`, checks if it exists already,
creates it and installs requirements otherwise. The virtualenvs are stored in a folder based
on the :class:`Arca` ``base_dir`` setting.
:param path: :class:`Path <pathlib.Path>` to the cloned repository.
"""
requirements_file = self.get_requirements_file(path)
venv_name = self.get_virtualenv_name(requirements_file)
venv_path = Path(self._arca.base_dir) / "venvs" / venv_name
if not venv_path.exists():
logger.info(f"Creating a venv in {venv_path}")
builder = EnvBuilder(with_pip=True)
builder.create(venv_path)
if requirements_file is not None:
logger.debug("Requirements file:")
logger.debug(requirements_file.read_text())
logger.info("Installing requirements from %s",
requirements_file)
process = subprocess.Popen([
str(venv_path / "bin" / "python3"), "-m", "pip", "install",
"-r",
str(requirements_file)
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
out_stream, err_stream = process.communicate(
timeout=self.requirements_timeout)
except subprocess.TimeoutExpired:
process.kill()
shutil.rmtree(venv_path, ignore_errors=True)
raise BuildTimeoutError(
f"Installing of requirements timeouted after "
f"{self.requirements_timeout} seconds.")
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
if process.returncode:
venv_path.rmdir()
raise BuildError("Unable to install requirements.txt",
extra_info={
"out_stream": out_stream,
"err_stream": err_stream,
"returncode": process.returncode
})
else:
logger.info(
"Requirements file not present in repo, empty venv it is.")
else:
logger.info(f"Venv already eixsts in {venv_path}")
return venv_path