def ensure_user_environment(user_requirements_txt_file): """ Set up user conda environment with required packages """ print("Setting up user environment...") conda.ensure_conda_env(USER_ENV_PREFIX) conda.ensure_conda_packages( USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. 'conda==4.5.8' ]) conda.ensure_pip_packages( USER_ENV_PREFIX, [ # JupyterHub + notebook package are base requirements for user environment 'jupyterhub==0.9.1', 'notebook==5.6.0', # Install additional notebook frontends! 'jupyterlab==0.32.1', 'nteract-on-jupyter==1.8.1', # nbgitpuller for easily pulling in Git repositories 'nbgitpuller==0.6.1' ]) if user_requirements_txt_file: # FIXME: This currently fails hard, should fail soft and not abort installer conda.ensure_pip_requirements(USER_ENV_PREFIX, user_requirements_txt_file)
def ensure_jupyterhub_package(prefix): """ Install JupyterHub into our conda environment if needed. We install all python packages from PyPI as much as possible in the hub environment. A lot of spawners & authenticators do not have conda-forge packages, but do have pip packages. Keeping all python packages in the hub environment be installed with pip prevents accidental mixing of python and conda packages! """ # Install pycurl. JupyterHub prefers pycurl over SimpleHTTPClient automatically # pycurl is generally more bugfree - see https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 # build-essential is also generally useful to everyone involved, and required for pycurl apt.install_packages( ['libssl-dev', 'libcurl4-openssl-dev', 'build-essential']) conda.ensure_pip_packages(prefix, ['pycurl==7.43.*']) conda.ensure_pip_packages( prefix, [ "jupyterhub==1.0.0", "jupyterhub-dummyauthenticator==0.3.1", "jupyterhub-systemdspawner==0.13", "jupyterhub-firstuseauthenticator==0.12", "jupyterhub-nativeauthenticator==0.0.4", "jupyterhub-ldapauthenticator==1.2.2", "jupyterhub-tmpauthenticator==0.6", "oauthenticator==0.8.2", ], ) traefik.ensure_traefik_binary(prefix)
def run_plugin_actions(plugin_manager, plugins): """ Run installer hooks defined in plugins """ hook = plugin_manager.hook # Install apt packages apt_packages = list(set(itertools.chain(*hook.tljh_extra_apt_packages()))) if apt_packages: logger.info('Installing {} apt packages collected from plugins: {}'.format( len(apt_packages), ' '.join(apt_packages) )) apt.install_packages(apt_packages) # Install conda packages conda_packages = list(set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) if conda_packages: logger.info('Installing {} conda packages collected from plugins: {}'.format( len(conda_packages), ' '.join(conda_packages) )) conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages) # Install pip packages pip_packages = list(set(itertools.chain(*hook.tljh_extra_user_pip_packages()))) if pip_packages: logger.info('Installing {} pip packages collected from plugins: {}'.format( len(pip_packages), ' '.join(pip_packages) )) conda.ensure_pip_packages(USER_ENV_PREFIX, pip_packages)
def ensure_jupyterhub_package(prefix): """ Install JupyterHub into our conda environment if needed. We install all python packages from PyPI as much as possible in the hub environment. A lot of spawners & authenticators do not have conda-forge packages, but do have pip packages. Keeping all python packages in the hub environment be installed with pip prevents accidental mixing of python and conda packages! """ # Install pycurl. JupyterHub prefers pycurl over SimpleHTTPClient automatically # pycurl is generally more bugfree - see https://github.com/jupyterhub/the-littlest-jupyterhub/issues/289 # build-essential is also generally useful to everyone involved, and required for pycurl apt.install_packages( ["libssl-dev", "libcurl4-openssl-dev", "build-essential"]) conda.ensure_pip_packages(prefix, ["pycurl==7.*"], upgrade=True) conda.ensure_pip_packages( prefix, [ "jupyterhub==1.*", "jupyterhub-systemdspawner==0.15.*", "jupyterhub-firstuseauthenticator==1.*", "jupyterhub-nativeauthenticator==1.*", "jupyterhub-ldapauthenticator==1.*", "jupyterhub-tmpauthenticator==0.6.*", "oauthenticator==14.*", "jupyterhub-idle-culler==1.*", "git+https://github.com/yuvipanda/jupyterhub-configurator@317759e17c8e48de1b1352b836dac2a230536dba", ], upgrade=True, ) traefik.ensure_traefik_binary(prefix)
def test_ensure_pip_packages(prefix): """ Test installing pip packages in conda environment """ conda.ensure_conda_packages(prefix, ['pip']) conda.ensure_pip_packages(prefix, ['numpy']) # Throws an error if this fails subprocess.check_call( [os.path.join(prefix, 'bin', 'python'), '-c', 'import numpy'])
def test_ensure_pip_packages(prefix): """ Test installing pip packages in conda environment """ conda.ensure_conda_packages(prefix, ["pip"]) conda.ensure_pip_packages(prefix, ["numpy"]) # Throws an error if this fails subprocess.check_call( [os.path.join(prefix, "bin", "python"), "-c", "import numpy"])
def ensure_user_environment(user_requirements_txt_file): """ Set up user conda environment with required packages """ logger.info("Setting up user environment...") miniconda_old_version = '4.5.4' miniconda_new_version = '4.7.10' conda_version = '4.5.8' # If no prior miniconda installation is found, we can install a newer version if not conda.check_miniconda_version( USER_ENV_PREFIX, miniconda_old_version) and not conda.check_miniconda_version( USER_ENV_PREFIX, miniconda_new_version): miniconda_installer_md5 = "1c945f2b3335c7b2b15130b1b2dc5cf4" conda_version = '4.8.1' logger.info('Downloading & setting up user environment...') with conda.download_miniconda_installer( miniconda_new_version, miniconda_installer_md5) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) conda.ensure_conda_packages( USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. 'conda==' + conda_version ]) conda.ensure_pip_packages( USER_ENV_PREFIX, [ # JupyterHub + notebook package are base requirements for user environment 'jupyterhub==1.0.0', 'notebook==5.7.8', # Install additional notebook frontends! 'jupyterlab==0.35.4', 'nteract-on-jupyter==2.0.7', # nbgitpuller for easily pulling in Git repositories 'nbgitpuller==0.6.1', # nbresuse to show people how much RAM they are using 'nbresuse==0.3.0', # Most people consider ipywidgets to be part of the core notebook experience 'ipywidgets==7.4.2', # Pin tornado 'tornado<6.0', ]) if user_requirements_txt_file: # FIXME: This currently fails hard, should fail soft and not abort installer conda.ensure_pip_requirements(USER_ENV_PREFIX, user_requirements_txt_file)
def ensure_jupyterhub_package(prefix): """ Install JupyterHub into our conda environment if needed. Conda constructor does not play well with conda-forge, so we can ship this with constructor """ # FIXME: Use fully deterministic package lists here conda.ensure_conda_packages(prefix, ['jupyterhub==0.9.0']) conda.ensure_pip_packages(prefix, [ 'jupyterhub-dummyauthenticator==0.3.1', 'jupyterhub-systemdspawner==0.9.12', ])
def setup_plugins(plugins=None): """ Install plugins & setup a pluginmanager """ # Install plugins if plugins: conda.ensure_pip_packages(HUB_ENV_PREFIX, plugins) # Set up plugin infrastructure pm = pluggy.PluginManager('tljh') pm.add_hookspecs(hooks) pm.load_setuptools_entrypoints('tljh') return pm
def run_plugin_actions(plugin_manager): """ Run installer hooks defined in plugins """ hook = plugin_manager.hook # Install apt packages apt_packages = list(set(itertools.chain(*hook.tljh_extra_apt_packages()))) if apt_packages: logger.info( "Installing {} apt packages collected from plugins: {}".format( len(apt_packages), " ".join(apt_packages))) apt.install_packages(apt_packages) # Install hub pip packages hub_pip_packages = list( set(itertools.chain(*hook.tljh_extra_hub_pip_packages()))) if hub_pip_packages: logger.info( "Installing {} hub pip packages collected from plugins: {}".format( len(hub_pip_packages), " ".join(hub_pip_packages))) conda.ensure_pip_packages( HUB_ENV_PREFIX, hub_pip_packages, upgrade=True, ) # Install conda packages conda_packages = list( set(itertools.chain(*hook.tljh_extra_user_conda_packages()))) if conda_packages: logger.info( "Installing {} user conda packages collected from plugins: {}". format(len(conda_packages), " ".join(conda_packages))) conda.ensure_conda_packages(USER_ENV_PREFIX, conda_packages) # Install pip packages user_pip_packages = list( set(itertools.chain(*hook.tljh_extra_user_pip_packages()))) if user_pip_packages: logger.info( "Installing {} user pip packages collected from plugins: {}". format(len(user_pip_packages), " ".join(user_pip_packages))) conda.ensure_pip_packages( USER_ENV_PREFIX, user_pip_packages, upgrade=True, ) # Custom post install actions hook.tljh_post_install()
def ensure_user_environment(user_requirements_txt_file): """ Set up user conda environment with required packages """ logger.info("Setting up user environment...") miniconda_version = '4.5.4' miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_version): logger.info('Downloading & setting up user environment...') with conda.download_miniconda_installer( miniconda_version, miniconda_installer_md5) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) # nbresuse needs psutil, which requires gcc apt.install_packages(['gcc']) conda.ensure_conda_packages( USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. 'conda==4.5.8' ]) conda.ensure_pip_packages( USER_ENV_PREFIX, [ # JupyterHub + notebook package are base requirements for user environment 'jupyterhub==0.9.4', 'notebook==5.7.0', # Install additional notebook frontends! 'jupyterlab==0.35.3', 'nteract-on-jupyter==1.9.12', # nbgitpuller for easily pulling in Git repositories 'nbgitpuller==0.6.1', # nbresuse to show people how much RAM they are using 'nbresuse==0.3.0', # Most people consider ipywidgets to be part of the core notebook experience 'ipywidgets==7.4.2', # Pin tornado 'tornado<6.0' ]) if user_requirements_txt_file: # FIXME: This currently fails hard, should fail soft and not abort installer conda.ensure_pip_requirements(USER_ENV_PREFIX, user_requirements_txt_file)
def ensure_jupyterhub_package(prefix): """ Install JupyterHub into our conda environment if needed. We install all python packages from PyPI as much as possible in the hub environment. A lot of spawners & authenticators do not have conda-forge packages, but do have pip packages. Keeping all python packages in the hub environment be installed with pip prevents accidental mixing of python and conda packages! """ conda.ensure_pip_packages(prefix, [ 'jupyterhub==0.9.4', 'jupyterhub-dummyauthenticator==0.3.1', 'jupyterhub-systemdspawner==0.11', 'jupyterhub-firstuseauthenticator==0.12', 'jupyterhub-nativeauthenticator==0.0.4', 'jupyterhub-ldapauthenticator==1.2.2', 'oauthenticator==0.8.0' ]) traefik.ensure_traefik_binary(prefix)
def ensure_user_environment(user_requirements_txt_file): """ Set up user conda environment with required packages """ logger.info("Setting up user environment...") miniconda_version = '4.5.4' miniconda_installer_md5 = "a946ea1d0c4a642ddf0c3a26a18bb16d" if not conda.check_miniconda_version(USER_ENV_PREFIX, miniconda_version): logger.info('Downloading & setting up user environment...') with conda.download_miniconda_installer( miniconda_version, miniconda_installer_md5) as installer_path: conda.install_miniconda(installer_path, USER_ENV_PREFIX) conda.ensure_conda_packages( USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. 'conda==4.5.8' ]) conda.ensure_pip_packages( USER_ENV_PREFIX, [ # JupyterHub + notebook package are base requirements for user environment 'jupyterhub==0.9.1', 'notebook==5.6.0', # Install additional notebook frontends! 'jupyterlab==0.32.1', 'nteract-on-jupyter==1.8.1', # nbgitpuller for easily pulling in Git repositories 'nbgitpuller==0.6.1' ]) if user_requirements_txt_file: # FIXME: This currently fails hard, should fail soft and not abort installer conda.ensure_pip_requirements(USER_ENV_PREFIX, user_requirements_txt_file)
def ensure_user_environment(): """ Set up user conda environment with required packages """ print("Setting up user environment...") conda.ensure_conda_env(USER_ENV_PREFIX) conda.ensure_conda_packages( USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. 'conda==4.5.8' ]) conda.ensure_pip_packages( USER_ENV_PREFIX, [ # JupyterHub + notebook package are base requirements for user environment 'jupyterhub==0.9.0', 'notebook==5.5.0', # Install additional notebook frontends! 'jupyterlab==0.32.1', 'nteract-on-jupyter==1.8.1' ])
ensure_jupyterhub_package(HUB_ENV_PREFIX) ensure_jupyterhub_service(HUB_ENV_PREFIX) user.ensure_group('jupyterhub-admins') user.ensure_group('jupyterhub-users') with open('/etc/sudoers.d/jupyterhub-admins', 'w') as f: # JupyterHub admins should have full passwordless sudo access f.write('%jupyterhub-admins ALL = (ALL) NOPASSWD: ALL\n') # `sudo -E` should preserve the $PATH we set. This allows # admins in jupyter terminals to do `sudo -E pip install <package>`, # `pip` is in the $PATH we set in jupyterhub_config.py to include the user conda env. f.write('Defaults exempt_group = jupyterhub-admins\n') conda.ensure_conda_env(USER_ENV_PREFIX) conda.ensure_conda_packages(USER_ENV_PREFIX, [ # Conda's latest version is on conda much more so than on PyPI. 'conda==4.5.4' ]) conda.ensure_pip_packages(USER_ENV_PREFIX, [ # JupyterHub + notebook package are base requirements for user environment 'jupyterhub==0.9.0', 'notebook==5.5.0', # Install additional notebook frontends! 'jupyterlab==0.32.1', 'nteract-on-jupyter==1.8.1' ])