def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path) -> Result:
"""
Gets a path to a Python executable by calling the abstract method
:meth:`get_image_for_repo <BaseRunInSubprocessBackend.get_image_for_repo>`
and runs the task using :class:`subprocess.Popen`
See :meth:`BaseBackend.run <arca.BaseBackend.run>` to see arguments description.
"""
python_path = self.get_or_create_environment(repo, branch, git_repo, repo_path)
task_filename, task_json = self.serialized_task(task)
task_path = Path(self._arca.base_dir, "tasks", task_filename)
task_path.parent.mkdir(parents=True, exist_ok=True)
task_path.write_text(task_json)
logger.info("Stored task definition at %s", task_path)
out_output = ""
err_output = ""
cwd = str(repo_path / self.cwd)
logger.info("Running at cwd %s", cwd)
try:
logger.debug("Running with python %s", python_path)
process = subprocess.Popen([python_path,
str(self.RUNNER),
str(task_path.resolve())],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
try:
out_stream, err_stream = process.communicate(timeout=task.timeout)
except subprocess.TimeoutExpired:
process.kill()
raise BuildTimeoutError(f"The task timeouted after {task.timeout} seconds.")
out_output = out_stream.decode("utf-8")
err_output = err_stream.decode("utf-8")
logger.debug("stdout output from the command")
logger.debug(out_output)
return Result(out_output)
except BuildError: # can be raised by :meth:`Result.__init__` or by timeout
raise
except Exception as e:
logger.exception(e)
raise BuildError("The build failed", extra_info={
"exception": e,
"out_output": out_output,
"err_output": err_output,
})
def check_docker_access(self):
""" Creates a :class:`DockerClient <docker.client.DockerClient>` for the instance and checks the connection.
:raise BuildError: If docker isn't accessible by the current user.
"""
try:
if self.client is None:
self.client = docker.from_env()
self.client.ping(
) # check that docker is running and user is permitted to access it
except ConnectionError as e:
logger.exception(e)
raise BuildError(
"Docker is not running or the current user doesn't have permissions to access docker."
)
def run(self, repo: str, branch: str, task: Task, git_repo: Repo,
repo_path: Path) -> Result:
""" Gets or builds an image for the repo, gets or starts a container for the image and runs the script.
:param repo: Repository URL
:param branch: Branch ane
:param task: :class:`Task` to run.
:param git_repo: :class:`Repo <git.repo.base.Repo>` of the cloned repository.
:param repo_path: :class:`Path <pathlib.Path>` to the cloned location.
"""
self.check_docker_access()
container_name = self.get_container_name(repo, branch, git_repo)
container = self.container_running(container_name)
if container is None:
image = self.get_image_for_repo(repo, branch, git_repo, repo_path)
container = self.start_container(image, container_name, repo_path)
task_filename, task_json = self.serialized_task(task)
container.put_archive(
"/srv/scripts", self.tar_task_definition(task_filename, task_json))
res = None
try:
command = ["timeout"]
if self.inherit_image:
if self.alpine_inherited or b"Alpine" in container.exec_run(
["cat", "/etc/issue"], tty=True).output:
self.alpine_inherited = True
command = ["timeout", "-t"]
command += [
str(task.timeout), "python", "/srv/scripts/runner.py",
f"/srv/scripts/{task_filename}"
]
logger.debug("Running command %s", " ".join(command))
res = container.exec_run(command, tty=True)
# 124 is the standard, 143 on alpine
if res.exit_code in {124, 143}:
raise BuildTimeoutError(
f"The task timeouted after {task.timeout} seconds.")
return Result(res.output)
except BuildError: # can be raised by :meth:`Result.__init__`
raise
except Exception as e:
logger.exception(e)
if res is not None:
logger.warning(res.output)
raise BuildError("The build failed",
extra_info={
"exception": e,
"output": res if res is None else res.output
})
finally:
if not self.keep_container_running:
container.kill(signal.SIGKILL)
else:
self._containers.add(container)
def get_or_build_image(self,
name: str,
tag: str,
dockerfile: Union[str, Callable[..., str]],
*,
pull=True,
build_context: Optional[Path] = None):
"""
A proxy for commonly built images, returns them from the local system if they exist, tries to pull them if
pull isn't disabled, otherwise builds them by the definition in ``dockerfile``.
:param name: Name of the image
:param tag: Image tag
:param dockerfile: Dockerfile text or a callable (no arguments) that produces Dockerfile text
:param pull: If the image is not present locally, allow pulling from registry (default is ``True``)
:param build_context: A path to a folder. If it's provided, docker will build the image in the context
of this folder. (eg. if ``ADD`` is needed)
"""
if self.image_exists(name, tag):
logger.info("Image %s:%s exists", name, tag)
return
elif pull:
logger.info("Trying to pull image %s:%s", name, tag)
try:
self.client.images.pull(name, tag=tag)
logger.info("The image %s:%s was pulled from registry", name,
tag)
return
except docker.errors.APIError:
logger.info(
"The image %s:%s can't be pulled, building locally.", name,
tag)
if callable(dockerfile):
dockerfile = dockerfile()
try:
if build_context is None:
fileobj = BytesIO(bytes(
dockerfile, "utf-8")) # required by the docker library
self.client.images.build(fileobj=fileobj, tag=f"{name}:{tag}")
else:
dockerfile_file = build_context / "dockerfile"
dockerfile_file.write_text(dockerfile)
self.client.images.build(path=str(build_context.resolve()),
dockerfile=dockerfile_file.name,
tag=f"{name}:{tag}")
dockerfile_file.unlink()
except docker.errors.BuildError as e:
for line in e.build_log:
if isinstance(line, dict) and line.get(
"errorDetail") and line["errorDetail"].get("code") in {
124, 143
}:
raise BuildTimeoutError(
f"Installing of requirements timeouted after "
f"{self.requirements_timeout} seconds.")
logger.exception(e)
raise BuildError(
"Building docker image failed, see extra info for details.",
extra_info={"build_log": e.build_log})
def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path):
""" Starts up a VM, builds an docker image and gets it to the VM, runs the script over SSH, returns result.
Stops the VM if ``keep_vm_running`` is not set.
"""
from fabric import api
from fabric.exceptions import CommandTimeout
# start up or get running VM
vm_location = self.get_vm_location()
self.ensure_vm_running(vm_location)
logger.info("Running with VM located at %s", vm_location)
# pushes the image to the registry so it can be pulled in the VM
self.check_docker_access() # init client
self.get_image_for_repo(repo, branch, git_repo, repo_path)
# getting things needed for execution over SSH
image_tag = self.get_image_tag(self.get_requirements_file(repo_path), self.get_dependencies())
image_name = self.use_registry_name
task_filename, task_json = self.serialized_task(task)
(vm_location / task_filename).write_text(task_json)
container_name = self.get_container_name(repo, branch, git_repo)
# setting up Fabric
api.env.hosts = [self.vagrant.user_hostname_port()]
api.env.key_filename = self.vagrant.keyfile()
api.env.disable_known_hosts = True # useful for when the vagrant box ip changes.
api.env.abort_exception = BuildError # raises SystemExit otherwise
api.env.shell = "/bin/sh -l -c"
if self.quiet:
api.output.everything = False
else:
api.output.everything = True
# executes the task
try:
res = api.execute(self.fabric_task,
container_name=container_name,
definition_filename=task_filename,
image_name=image_name,
image_tag=image_tag,
repository=str(repo_path.relative_to(Path(self._arca.base_dir).resolve() / 'repos')),
timeout=task.timeout)
return Result(res[self.vagrant.user_hostname_port()].stdout)
except CommandTimeout:
raise BuildTimeoutError(f"The task timeouted after {task.timeout} seconds.")
except BuildError: # can be raised by :meth:`Result.__init__`
raise
except Exception as e:
logger.exception(e)
raise BuildError("The build failed", extra_info={
"exception": e
})
finally:
# stops or destroys the VM if it should not be kept running
if not self.keep_vm_running:
if self.destroy:
self.vagrant.destroy()
shutil.rmtree(self.vagrant.root, ignore_errors=True)
self.vagrant = None
else:
self.vagrant.halt()