def push_to_registry(self, image, image_tag: str):
""" Pushes a local image to a registry based on the ``use_registry_name`` setting.
:type image: docker.models.images.Image
:raise PushToRegistryError: If the push fails.
"""
# already tagged, so it's already pushed
if f"{self.use_registry_name}:{image_tag}" in image.tags:
return
image.tag(self.use_registry_name, image_tag)
result = self.client.images.push(self.use_registry_name, image_tag)
result = result.strip() # remove empty line at the end of output
# the last can have one of two outputs, either
# {"progressDetail":{},"aux":{"Tag":"<tag>","Digest":"sha256:<hash>","Size":<size>}}
# when the push is successful, or
# {"errorDetail": {"message":"<error_msg>"},"error":"<error_msg>"}
# when the push is not successful
last_line = json.loads(result.split("\n")[-1])
if "error" in last_line:
self.client.images.remove(f"{self.use_registry_name}:{image_tag}")
raise PushToRegistryError(
f"Push of the image failed because of: {last_line['error']}",
full_output=result)
logger.info("Pushed image to registry %s:%s", self.use_registry_name,
image_tag)
logger.debug("Info:\n%s", result)
def get_install_requirements_dockerfile(
self, name: str, tag: str, repo_path: Path,
requirements_option: RequirementsOptions) -> str:
"""
Returns the content of a Dockerfile that will install requirements based on the repository,
prioritizing Pipfile or Pipfile.lock and falling back on requirements.txt files
"""
if requirements_option == RequirementsOptions.requirements_txt:
target_file = "requirements.txt"
requirements_files = [repo_path / self.requirements_location]
install_cmd = "pip"
cmd_arguments = "install -r /srv/requirements.txt"
elif requirements_option == RequirementsOptions.pipfile:
target_file = ""
requirements_files = [
repo_path / self.pipfile_location / "Pipfile",
repo_path / self.pipfile_location / "Pipfile.lock"
]
install_cmd = "pipenv"
cmd_arguments = "install --system --ignore-pipfile --deploy"
else:
raise ValueError("Invalid requirements_option")
dockerfile = self.INSTALL_REQUIREMENTS.format(
name=name,
tag=tag,
timeout=self.requirements_timeout,
target_file=target_file,
requirements_files=" ".join(
str(x.relative_to(repo_path.parent))
for x in requirements_files),
cmd_arguments=cmd_arguments,
install_cmd=install_cmd)
logger.debug("Installing Python requirements with Dockerfile: %s",
dockerfile)
return dockerfile
def _pip_action(action, package):
if action not in ["install", "uninstall"]:
raise ValueError(f"{action} is invalid value for action")
cmd = [sys.executable, "-m", "pip", action]
if action == "uninstall":
cmd += ["-y"]
cmd += [package]
logger.info("Installing requirements with command: %s", cmd)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out_stream, err_stream = process.communicate()
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path) -> Result:
"""
Gets a path to a Python executable by calling the abstract method
:meth:`get_image_for_repo <BaseRunInSubprocessBackend.get_image_for_repo>`
and runs the task using :class:`subprocess.Popen`
See :meth:`BaseBackend.run <arca.BaseBackend.run>` to see arguments description.
"""
python_path = self.get_or_create_environment(repo, branch, git_repo, repo_path)
task_filename, task_json = self.serialized_task(task)
task_path = Path(self._arca.base_dir, "tasks", task_filename)
task_path.parent.mkdir(parents=True, exist_ok=True)
task_path.write_text(task_json)
logger.info("Stored task definition at %s", task_path)
out_output = ""
err_output = ""
cwd = str(repo_path / self.cwd)
logger.info("Running at cwd %s", cwd)
try:
logger.debug("Running with python %s", python_path)
process = subprocess.Popen([python_path,
str(self.RUNNER),
str(task_path.resolve())],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
try:
out_stream, err_stream = process.communicate(timeout=task.timeout)
except subprocess.TimeoutExpired:
process.kill()
raise BuildTimeoutError(f"The task timeouted after {task.timeout} seconds.")
out_output = out_stream.decode("utf-8")
err_output = err_stream.decode("utf-8")
logger.debug("stdout output from the command")
logger.debug(out_output)
return Result(out_output)
except BuildError: # can be raised by :meth:`Result.__init__` or by timeout
raise
except Exception as e:
logger.exception(e)
raise BuildError("The build failed", extra_info={
"exception": e,
"out_output": out_output,
"err_output": err_output,
})
def install_requirements(self, *, path: Optional[Path] = None, requirements: Optional[Iterable[str]] = None,
_action: str = "install"):
"""
Installs requirements, either from a file or from a iterable of strings.
:param path: :class:`Path <pathlib.Path>` to a ``requirements.txt`` file. Has priority over ``requirements``.
:param requirements: A iterable of strings of requirements to install.
:param _action: For testing purposes, can be either ``install`` or ``uninstall``
:raise BuildError: If installing fails.
:raise ValueError: If both ``file`` and ``requirements`` are undefined.
:raise ValueError: If ``_action`` not ``install`` or ``uninstall``.
"""
if _action not in ["install", "uninstall"]:
raise ValueError(f"{_action} is invalid value for _action")
cmd = [sys.executable, "-m", "pip", _action]
if _action == "uninstall":
cmd += ["-y"]
if path is not None:
cmd += ["-r", str(path)]
elif requirements is not None:
cmd += list(requirements)
else:
raise ValueError("Either path or requirements has to be provided")
logger.info("Installing requirements with command: %s", cmd)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
out_stream, err_stream = process.communicate(timeout=self.requirements_timeout)
except subprocess.TimeoutExpired:
process.kill()
raise BuildTimeoutError(f"Installing of requirements timeouted after {self.requirements_timeout} seconds.")
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
if process.returncode:
raise BuildError(f"Unable to {_action} requirements from the target repository", extra_info={
"out_stream": out_stream,
"err_stream": err_stream,
"returncode": process.returncode
})
def get_or_create_venv(self, path: Path) -> Path:
"""
Gets the location of the virtualenv from :meth:`get_virtualenv_path`, checks if it exists already,
creates it and installs requirements otherwise. The virtualenvs are stored in a folder based
on the :class:`Arca` ``base_dir`` setting.
:param path: :class:`Path <pathlib.Path>` to the cloned repository.
"""
requirements_option, requirements_hash = self.get_requirements_information(
path)
venv_path = self.get_virtualenv_path(requirements_option,
requirements_hash)
if not venv_path.exists():
logger.info(f"Creating a venv in {venv_path}")
builder = EnvBuilder(with_pip=True)
builder.create(venv_path)
shell = False
cmd = None
cwd = None
if requirements_option == RequirementsOptions.pipfile:
cmd = [
"source", (str(venv_path / "bin" / "activate")), "&&",
"pipenv", "install", "--deploy", "--ignore-pipfile"
]
cmd = " ".join(cmd)
cwd = path / self.pipfile_location
shell = True
elif requirements_option == RequirementsOptions.requirements_txt:
requirements_file = path / self.requirements_location
logger.debug("Requirements file:")
logger.debug(requirements_file.read_text())
logger.info("Installing requirements from %s",
requirements_file)
cmd = [
str(venv_path / "bin" / "python3"), "-m", "pip", "install",
"-r",
shlex.quote(str(requirements_file))
]
if cmd is not None:
logger.info("Running Popen cmd %s, with shell %s", cmd, shell)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=shell,
cwd=cwd)
try:
out_stream, err_stream = process.communicate(
timeout=self.requirements_timeout)
except subprocess.TimeoutExpired:
process.kill()
logger.warning(
"The install command timed out, deleting the virtualenv"
)
shutil.rmtree(str(venv_path), ignore_errors=True)
raise BuildTimeoutError(
f"Installing of requirements timeouted after "
f"{self.requirements_timeout} seconds.")
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
if process.returncode:
logger.warning(
"The install command failed, deleting the virtualenv")
shutil.rmtree(str(venv_path), ignore_errors=True)
raise BuildError("Unable to install requirements.txt",
extra_info={
"out_stream": out_stream,
"err_stream": err_stream,
"returncode": process.returncode
})
else:
logger.info(
"Requirements file not present in repo, empty venv it is.")
else:
logger.info(f"Venv already exists in {venv_path}")
return venv_path
def run(self, repo: str, branch: str, task: Task, git_repo: Repo,
repo_path: Path) -> Result:
""" Gets or builds an image for the repo, gets or starts a container for the image and runs the script.
:param repo: Repository URL
:param branch: Branch ane
:param task: :class:`Task` to run.
:param git_repo: :class:`Repo <git.repo.base.Repo>` of the cloned repository.
:param repo_path: :class:`Path <pathlib.Path>` to the cloned location.
"""
self.check_docker_access()
container_name = self.get_container_name(repo, branch, git_repo)
container = self.container_running(container_name)
if container is None:
image = self.get_image_for_repo(repo, branch, git_repo, repo_path)
container = self.start_container(image, container_name, repo_path)
task_filename, task_json = self.serialized_task(task)
container.put_archive(
"/srv/scripts", self.tar_task_definition(task_filename, task_json))
res = None
try:
command = ["timeout"]
if self.inherit_image:
if self.alpine_inherited or b"Alpine" in container.exec_run(
["cat", "/etc/issue"], tty=True).output:
self.alpine_inherited = True
command = ["timeout", "-t"]
command += [
str(task.timeout), "python", "/srv/scripts/runner.py",
f"/srv/scripts/{task_filename}"
]
logger.debug("Running command %s", " ".join(command))
res = container.exec_run(command, tty=True)
# 124 is the standard, 143 on alpine
if res.exit_code in {124, 143}:
raise BuildTimeoutError(
f"The task timeouted after {task.timeout} seconds.")
return Result(res.output)
except BuildError: # can be raised by :meth:`Result.__init__`
raise
except Exception as e:
logger.exception(e)
if res is not None:
logger.warning(res.output)
raise BuildError("The build failed",
extra_info={
"exception": e,
"output": res if res is None else res.output
})
finally:
if not self.keep_container_running:
container.kill(signal.SIGKILL)
else:
self._containers.add(container)
def get_or_create_venv(self, path: Path) -> Path:
"""
Gets the name of the virtualenv from :meth:`get_virtualenv_name`, checks if it exists already,
creates it and installs requirements otherwise. The virtualenvs are stored in a folder based
on the :class:`Arca` ``base_dir`` setting.
:param path: :class:`Path <pathlib.Path>` to the cloned repository.
"""
requirements_file = self.get_requirements_file(path)
venv_name = self.get_virtualenv_name(requirements_file)
venv_path = Path(self._arca.base_dir) / "venvs" / venv_name
if not venv_path.exists():
logger.info(f"Creating a venv in {venv_path}")
builder = EnvBuilder(with_pip=True)
builder.create(venv_path)
if requirements_file is not None:
logger.debug("Requirements file:")
logger.debug(requirements_file.read_text())
logger.info("Installing requirements from %s",
requirements_file)
process = subprocess.Popen([
str(venv_path / "bin" / "python3"), "-m", "pip", "install",
"-r",
str(requirements_file)
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
out_stream, err_stream = process.communicate(
timeout=self.requirements_timeout)
except subprocess.TimeoutExpired:
process.kill()
shutil.rmtree(venv_path, ignore_errors=True)
raise BuildTimeoutError(
f"Installing of requirements timeouted after "
f"{self.requirements_timeout} seconds.")
out_stream = out_stream.decode("utf-8")
err_stream = err_stream.decode("utf-8")
logger.debug("Return code is %s", process.returncode)
logger.debug(out_stream)
logger.debug(err_stream)
if process.returncode:
venv_path.rmdir()
raise BuildError("Unable to install requirements.txt",
extra_info={
"out_stream": out_stream,
"err_stream": err_stream,
"returncode": process.returncode
})
else:
logger.info(
"Requirements file not present in repo, empty venv it is.")
else:
logger.info(f"Venv already eixsts in {venv_path}")
return venv_path
def get_requirements_hash(self, requirements_file: Path) -> str:
""" Returns an SHA1 hash of the contents of the ``requirements_path``.
"""
logger.debug("Hashing: %s%s", requirements_file.read_text(), arca.__version__)
return hashlib.sha256(bytes(requirements_file.read_text() + arca.__version__, "utf-8")).hexdigest()