def ensure_jupyterlab_extensions():
"""
Install the JupyterLab extensions we want.
"""
extensions = [
# We don't pin versions here, since labextension will find something
# appropriate for our version of jupyterlab
'@jupyter-widgets/jupyterlab-manager'
]
install_options = [
'--no-build' # do not build extension at install time. Will build later
]
utils.run_subprocess([
os.path.join(USER_ENV_PREFIX, 'bin/jupyter'),
'labextension',
'install'
] + extensions + install_options)
# Build all the lab extensions in one go using jupyter lab build command
build_options = [
'--minimize=False',
'--dev-build=False'
]
utils.run_subprocess([
os.path.join(USER_ENV_PREFIX, 'bin/jupyter'),
'lab',
'build'
] + build_options)
def install_packages(packages):
"""
Install debian packages
"""
# Check if an apt-get update is required
if len(os.listdir('/var/lib/apt/lists')) == 0:
utils.run_subprocess(['apt-get', 'update', '--yes'])
utils.run_subprocess(['apt-get', 'install', '--yes'] + packages)
def install_miniconda(installer_path, prefix):
"""
Install miniconda with installer at installer_path under prefix
"""
utils.run_subprocess(
['/bin/bash', installer_path, '-u', '-b', '-p', prefix])
# fix permissions on initial install
# a few files have the wrong ownership and permissions initially
# when the installer is run as root
fix_permissions(prefix)
def trust_gpg_key(key):
"""
Trust given GPG public key.
key is a GPG public key (bytes) that can be passed to apt-key add via stdin.
"""
# If gpg2 doesn't exist, install it.
if not os.path.exists('/usr/bin/gpg2'):
install_packages(['gnupg2'])
utils.run_subprocess(['apt-key', 'add', '-'], input=key)
def ensure_jupyterlab_extensions():
"""
Install the JupyterLab extensions we want.
"""
extensions = [
'@jupyterlab/hub-extension', '@jupyter-widgets/jupyterlab-manager'
]
utils.run_subprocess([
os.path.join(USER_ENV_PREFIX, 'bin/jupyter'), 'labextension', 'install'
] + extensions)
def trust_gpg_key(key):
"""
Trust given GPG public key.
key is a GPG public key (bytes) that can be passed to apt-key add via stdin.
"""
# If gpg2 doesn't exist, install it.
if not os.path.exists("/usr/bin/gpg2"):
install_packages(["gnupg2"])
utils.run_subprocess(["apt-key", "add", "-"], input=key)
def install_packages(packages):
"""
Install debian packages
"""
# Check if an apt-get update is required
if len(os.listdir("/var/lib/apt/lists")) == 0:
utils.run_subprocess(["apt-get", "update", "--yes"])
env = os.environ.copy()
# Stop apt from asking questions!
env["DEBIAN_FRONTEND"] = "noninteractive"
utils.run_subprocess(["apt-get", "install", "--yes"] + packages, env=env)
def ensure_pip_requirements(prefix, requirements_path):
"""
Ensure pip packages from given requirements_path are installed in given conda prefix.
requirements_path can be a file or a URL.
"""
abspath = os.path.abspath(prefix)
pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip']
utils.run_subprocess(pip_executable + ['install', '-r', requirements_path])
fix_permissions(prefix)
def ensure_pip_packages(prefix, packages, upgrade=False):
"""
Ensure pip packages are installed in the given conda prefix.
"""
abspath = os.path.abspath(prefix)
pip_executable = [os.path.join(abspath, "bin", "python"), "-m", "pip"]
pip_cmd = pip_executable + ["install"]
if upgrade:
pip_cmd.append("--upgrade")
utils.run_subprocess(pip_cmd + packages)
fix_permissions(prefix)
def install_packages(packages):
"""
Install debian packages
"""
# Check if an apt-get update is required
if len(os.listdir('/var/lib/apt/lists')) == 0:
utils.run_subprocess(['apt-get', 'update', '--yes'])
env = os.environ.copy()
# Stop apt from asking questions!
env['DEBIAN_FRONTEND'] = 'noninteractive'
utils.run_subprocess(['apt-get', 'install', '--yes'] + packages, env=env)
def fix_permissions(prefix):
"""Fix permissions in the install prefix
For all files in the prefix, ensure that:
- everything is owned by current user:group
- nothing is world-writeable
Run after each install command.
"""
utils.run_subprocess(
["chown", "-R", "{}:{}".format(os.getuid(), os.getgid()), prefix])
utils.run_subprocess(["chmod", "-R", "o-w", prefix])
def ensure_pip_packages(prefix, packages):
"""
Ensure pip packages are installed in the given conda prefix.
"""
abspath = os.path.abspath(prefix)
pip_executable = [os.path.join(abspath, 'bin', 'python'), '-m', 'pip']
utils.run_subprocess(pip_executable + [
'install',
'--no-cache-dir',
] + packages)
fix_permissions(prefix)
def ensure_pip_requirements(prefix, requirements_path, upgrade=False):
"""
Ensure pip packages from given requirements_path are installed in given conda prefix.
requirements_path can be a file or a URL.
"""
abspath = os.path.abspath(prefix)
pip_executable = [os.path.join(abspath, "bin", "python"), "-m", "pip"]
pip_cmd = pip_executable + ["install"]
if upgrade:
pip_cmd.append("--upgrade")
utils.run_subprocess(pip_cmd + ["--requirement", requirements_path])
fix_permissions(prefix)
def add_source(name, source_url, section):
"""
Add a debian package source.
distro is determined from /etc/os-release
"""
# lsb_release is not installed in most docker images by default
distro = subprocess.check_output(['/bin/bash', '-c', 'source /etc/os-release && echo ${VERSION_CODENAME}'], stderr=subprocess.STDOUT).decode().strip()
line = f'deb {source_url} {distro} {section}'
with open(os.path.join('/etc/apt/sources.list.d/', name + '.list'), 'a+') as f:
# Write out deb line only if it already doesn't exist
if f.read() != line:
f.seek(0)
f.write(line)
f.truncate()
utils.run_subprocess(['apt-get', 'update', '--yes'])
def ensure_jupyterlab_extensions():
"""
Install the JupyterLab extensions we want.
"""
extensions = [
'@jupyter-widgets/[email protected]' # for jupyterlab 1.2.x
]
install_options = [
'--no-build' # do not build extension at install time. Will build later
]
utils.run_subprocess([
os.path.join(USER_ENV_PREFIX, 'bin/jupyter'), 'labextension', 'install'
] + extensions + install_options)
# Build all the lab extensions in one go using jupyter lab build command
build_options = ['--minimize=False', '--dev-build=False']
utils.run_subprocess(
[os.path.join(USER_ENV_PREFIX, 'bin/jupyter'), 'lab', 'build'] +
build_options)
def add_source(name, source_url, section):
"""
Add a debian package source.
distro is determined from /etc/os-release
"""
# lsb_release is not installed in most docker images by default
distro = (
subprocess.check_output(
["/bin/bash", "-c", "source /etc/os-release && echo ${VERSION_CODENAME}"],
stderr=subprocess.STDOUT,
)
.decode()
.strip()
)
line = f"deb {source_url} {distro} {section}\n"
with open(os.path.join("/etc/apt/sources.list.d/", name + ".list"), "a+") as f:
# Write out deb line only if it already doesn't exist
f.seek(0)
if line not in f.read():
f.write(line)
f.truncate()
utils.run_subprocess(["apt-get", "update", "--yes"])
def test_run_subprocess_exception(mocker):
logger = logging.getLogger('tljh')
mocker.patch.object(logger, 'error')
with pytest.raises(subprocess.CalledProcessError):
utils.run_subprocess(['/bin/bash', '-c', 'echo error; exit 1'])
logger.error.assert_called_with('error\n')
def test_run_subprocess_exception(mocker):
logger = logging.getLogger("tljh")
mocker.patch.object(logger, "error")
with pytest.raises(subprocess.CalledProcessError):
utils.run_subprocess(["/bin/bash", "-c", "echo error; exit 1"])
logger.error.assert_called_with("error\n")
def test_run_subprocess(mocker):
logger = logging.getLogger("tljh")
mocker.patch.object(logger, "debug")
utils.run_subprocess(["/bin/bash", "-c", "echo success"])
logger.debug.assert_called_with("success\n")
def test_run_subprocess(mocker):
logger = logging.getLogger('tljh')
mocker.patch.object(logger, 'debug')
utils.run_subprocess(['/bin/bash', '-c', 'echo success'])
logger.debug.assert_called_with('success\n')