def exists(host: Driver, path: str):
if host.exec(f"test -e {path}", assert_ok=False).return_code == 0:
return True
elif host.exec(f"test -L {path}", assert_ok=False).return_code == 0:
return True
else:
return False
def pyenv_install(host: Driver, python_version: str):
install_pyenv(host)
if python_version in host.exec(
"~/.pyenv/bin/pyenv versions --bare").stdout.decode():
return
host.exec(f"~/.pyenv/bin/pyenv install {python_version}")
def npm_install(host: Driver, node_version: str, cwd: str):
"""
Run `npm install` in the directory `cwd`, after executing `nvm use
{node_version}`.
"""
host.exec_as_script("\n".join([
*nvm_init_script, f"nvm use {node_version}", f"cd {cwd}", "npm install"
]))
def ssh_keyscan(host: Driver, hostname: str):
directory(host, "~/.ssh")
if not is_regular_file(host, "~/.ssh/known_hosts"):
host.exec("touch ~/.ssh/known_hosts")
if host.exec(f"ssh-keygen -F {hostname}",
assert_ok=False).return_code != 0:
host.exec(f'ssh-keyscan {hostname} >> ~/.ssh/known_hosts')
def nvm_install(host: Driver, node_version: str):
"""
Use nvm to install version `node_version` of node. Eg.
`nvm_install("v13.7.0")`. Any version that appears in the list output when
you run `nvm ls` should work.
"""
install_nvm(host)
host.exec_as_script("\n".join(
[*nvm_init_script, f"nvm install {node_version}"]))
def link(host: Driver, target: str, link_name: str):
if host.exec(
f"[ $(readlink -f $(realpath {link_name})) = $(realpath {target}) ]",
assert_ok=False).return_code == 0:
return
if exists(host, link_name):
host.exec(f"rm {link_name}")
host.exec(f"ln -s {target} {link_name}")
def pyenv_virtualenv(host: Driver, python_version: str, virtualenv_name: str):
install_pyenv(host)
if (f"{python_version}/envs/{virtualenv_name}"
in host.exec("~/.pyenv/bin/pyenv virtualenvs --bare",
echo_stdout=False).stdout.decode().splitlines()):
return
pyenv_install(host, python_version)
host.exec(
f"~/.pyenv/bin/pyenv virtualenv {python_version} {virtualenv_name}")
def pip_install(host: Driver,
python_version: str,
virtualenv_name: str,
what_to_install: str,
editable: bool = False) -> None:
host.exec_as_script("\n".join([
'export PATH="$HOME/.pyenv/bin:$PATH"', 'eval "$(pyenv init -)"',
'eval "$(pyenv virtualenv-init -)"',
f"pyenv activate {virtualenv_name}",
f"pip install {'-e' if editable else ''} {what_to_install}"
]))
def install_nvm(host: Driver) -> None:
"""
Download and install nvm. Add the commands for auto-loading it to the rc
file for the particular shell (if it's a shell we recognise).
"""
stdout = host.exec("test -f ~/.nvm/nvm.sh > /dev/null; echo $?").stdout
if stdout.decode().strip() == "0":
return
if host.platform == "ubuntu":
apt_install(host, ["curl"])
elif host.platform == "arch":
pacman_install(host, ["curl"])
else:
print("Neither ubuntu nor arch, let's continue and hope we have curl")
host.exec(
"curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.36.0/install.sh | bash"
)
host.exec_as_script("\n".join(nvm_init_script))
rc_file = get_shell_rc_file(host)
if rc_file is not None:
if host.exec(f"grep NVM_DIR {rc_file}").return_code != 0:
for i, line in enumerate(nvm_init_script):
host.exec_as_script("\n".join([
f"cat >> {rc_file} <<EOF",
*nvm_init_script,
"EOF",
]))
def pyenv_install_rc(host: Driver):
lines = [
'export PATH="$HOME/.pyenv/bin:$PATH"',
'eval "$(pyenv init -)"',
'eval "$(pyenv virtualenv-init -)"',
]
rc_file = get_shell_rc_file(host)
if rc_file is not None:
if host.exec(f"grep pyenv {rc_file}",
assert_ok=False).return_code != 0:
host.exec_as_script("\n".join([
f"cat >> {rc_file} <<'EOF'",
*lines,
"EOF",
]))
def install_pyenv(host: Driver):
if is_regular_file(host, "~/.pyenv/bin/pyenv"):
return
# If you don't have tzdata setup, consult the web service at
# ipapi.co/timezone to get the correct time zone and put it in
# /etc/localtime. Otherwise installing the following packages would
# interactively prompt for your time zone.
# https://stackoverflow.com/a/63153978/223486
if not exists(host, "/etc/localtime"):
host.exec(
"ln -snf /usr/share/zoneinfo/$(curl https://ipapi.co/timezone) /etc/localtime"
)
if host.platform == "ubuntu":
apt_install(
host,
[
# Required to install pyenv itself
"curl",
"git",
# Required to use pyenv to install Python versions
"build-essential",
"libssl-dev",
"zlib1g-dev",
"libbz2-dev",
"libreadline-dev",
"libsqlite3-dev",
"llvm",
"libncurses5-dev",
"libncursesw5-dev",
"xz-utils",
"tk-dev",
"libffi-dev",
"liblzma-dev",
# Possibly was required for earlier versions of Ubuntu? Doesn't seem to
# exist anymore and we seem to be fine without it.
#"python-openssl",
])
elif host.platform == "arch":
pacman_install(host, ["git", "base-devel"])
else:
raise ValueError("Don't know what to install on this platform")
host.exec("curl https://pyenv.run |bash")
def apt_install(host: Driver, packages: List[str]):
installed_packages = host.exec(
"dpkg --get-selections |grep -v deinstall |cut -f 1",
echo_stdout=False,
echo_stderr=False,
).stdout.splitlines()
packages_to_install = set(packages) - set(installed_packages)
if packages_to_install:
try:
host.exec(f"apt-get install -y {' '.join(packages_to_install)}",
sudo=host.has_sudo)
except CommandFailed as e:
if "Unable to locate package" in e.stderr.decode():
apt_update(host)
host.exec(
f"apt-get install -y {' '.join(packages_to_install)}",
sudo=host.has_sudo)
def install_python_app(host: Driver, python_version: str, virtualenv_name: str,
install_cmd: str, binaries: List[str]):
pyenv_virtualenv(host, python_version, virtualenv_name)
host.exec_as_script(f"""
export PATH="$HOME/.pyenv/bin:$PATH"
eval "$(pyenv init -)"
eval "$(pyenv virtualenv-init -)"
pyenv activate {python_version}/envs/{virtualenv_name}
{install_cmd}
""")
for b in binaries:
link(
host,
link_name=f"~/bin/{b}",
target=f"{virtualenv_dir(python_version, virtualenv_name)}/bin/{b}"
)
def pacman_install(host: Driver, packages: List[str]):
try:
host.exec(f"pacman -S {' '.join(packages)} --noconfirm", sudo=True)
except CommandFailed as e:
if "failed to initialize" in e.stderr.decode():
# https://www.reddit.com/r/archlinux/comments/lek2ba/arch_linux_on_docker_ci_could_not_find_or_read/
# https://github.com/qutebrowser/qutebrowser/commit/478e4de7bd1f26bebdcdc166d5369b2b5142c3e2
host.exec(
'''patched_glibc=glibc-linux4-2.33-4-x86_64.pkg.tar.zst && curl -LO "https://repo.archlinuxcn.org/x86_64/$patched_glibc" && bsdtar -C / -xvf "$patched_glibc"'''
)
host.exec("pacman -Syu --noconfirm", sudo=True)
host.exec(f"pacman -S {' '.join(packages)} --noconfirm", sudo=True)
def repo(host: Driver,
src: str,
dest: str,
ssh_key_path: Optional[str] = None):
install_git(host)
directory(host, dest)
repo_path = src.split("/")[-1]
assert repo_path.endswith(".git")
repo_name = repo_path[0:-len(".git")]
if is_directory(host, f"{dest}/{repo_name}"):
return
# Attempt to parse the domain from the git repository (stolen from pyinfra)
domain = re.match(r'^[a-zA-Z0-9]+@([0-9a-zA-Z\.\-]+)', src)
if domain:
ssh_keyscan(host, domain.group(1))
if ssh_key_path is not None:
ssh_command_param = f'ssh -i {ssh_key_path}'
host.exec(
f"cd {dest} && git -c core.sshCommand='{ssh_command_param}' clone {src}"
)
else:
host.exec(f"cd {dest} && git clone {src}")
if ssh_key_path is not None:
host.exec(
f"cd {dest}/{repo_name}; git config core.sshCommand '{ssh_command_param}'"
)
def install_mongo(host: Driver):
host.exec(
"wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add -",
sudo=host.has_sudo)
host.exec(
'echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list',
sudo=host.has_sudo)
host.exec("apt-get update -y", sudo=host.has_sudo)
apt_install(host, ["mongodb-org"])
def install_docker_compose(host: Driver):
if is_regular_file(host, "/usr/local/bin/docker-compose"):
return
apt_install(host, ["curl"])
host.exec(
'curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)"'
' -o /usr/local/bin/docker-compose',
sudo=host.has_sudo)
user = host.exec("whoami").stdout.decode().strip()
host.exec(f"chown {user} /usr/local/bin/docker-compose",
sudo=host.has_sudo)
host.exec("chmod +x /usr/local/bin/docker-compose")
def is_regular_file(host: Driver, path: str):
return host.exec(f"test -f {path}", assert_ok=False).return_code == 0
def apt_update(host: Driver):
host.exec("apt-get update -y", sudo=host.has_sudo)
def init_apt(host: Driver):
if host.exec("ls /var/lib/apt/lists/* > /dev/null",
assert_ok=False).return_code != 0:
apt_update(host)
def install_docker(host: Driver, client_only: bool = False):
if host.exec("which docker", assert_ok=False).return_code == 0:
return
if host.platform == "ubuntu":
apt_install(host, [
"apt-transport-https", "ca-certificates", "curl", "gnupg-agent",
"software-properties-common"
])
host.exec(
"sh -c 'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -'",
sudo=True)
ubuntu_codename = get_ubuntu_codename(host)
if ubuntu_codename == "groovy":
# Currenty groovy (Ubuntu 20.10) doesn't have packages in this repo,
# but we can fall back to focal and everything seems to work.
ubuntu_codename = "focal"
host.exec(
f'add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu {ubuntu_codename} stable" -y',
sudo=True)
host.exec("apt-get update -y", sudo=True)
if client_only:
apt_install(host, ["docker-ce-cli"])
else:
apt_install(host, ["docker-ce", "docker-ce-cli", "containerd.io"])
host.exec("service docker start", sudo=True)
if host.exec("whoami").stdout.decode().strip() != "root":
host.exec("usermod -aG docker $USER", sudo=True)
elif host.platform == "arch":
pacman_install(host, ["docker"])
if not client_only:
host.exec("systemctl start docker.service", sudo=True)
host.exec("systemctl start docker.service", sudo=True)
else:
raise ValueError("Don't know how to install Docker on this platform")
def get_ubuntu_codename(host: Driver):
"""
Return "focal", "groovy" etc.
"""
return host.exec("lsb_release -cs").stdout.decode().strip()
def directory(host: Driver, path: str):
if not is_directory(host, path):
host.exec(f"mkdir -p {path}")
def is_directory(host: Driver, path: str):
return host.exec(f"test -d {path}", assert_ok=False).return_code == 0