def activate_venv(): """ Activate the venv if enabled in ``layer.yaml``. This is handled automatically for normal hooks, but actions might need to invoke this manually, using something like: # Load modules from $JUJU_CHARM_DIR/lib import sys sys.path.append('lib') from charms.layer.basic import activate_venv activate_venv() This will ensure that modules installed in the charm's virtual environment are available to the action. """ from charms.layer import options venv = os.path.abspath('../.venv') vbin = os.path.join(venv, 'bin') vpy = os.path.join(vbin, 'python') use_venv = options.get('basic', 'use_venv') if use_venv and '.venv' not in sys.executable: # activate the venv os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']]) reload_interpreter(vpy) layer.patch_options_interface() layer.import_layer_libs()
def reconfigure(): cfg = options.get("venv") if "packages" in cfg: if helpers.data_changed("venv-packages", cfg["packages"]): clear_flag("venv.ready") if helpers.data_changed("venv-name", ENV_NAME): clear_flag("venv.active")
def configure(opts={}): layer_opts = options.get('logrotate') context = {} context['options'] = { **layer_opts, **opts } #merge options, param opts overwrite target = LOGROTATE_DIR + service_name() + '.conf' if context['options']: log("Rendering {} with {}".format(target, context)) render(source=LOGROTATE_TEMPLATE, target=target, context=context)
def bootstrap_charm_deps(): """ Set up the base charm dependencies so that the reactive system can run. """ # execd must happen first, before any attempt to install packages or # access the network, because sites use this hook to do bespoke # configuration and install secrets so the rest of this bootstrap # and the charm itself can actually succeed. This call does nothing # unless the operator has created and populated $JUJU_CHARM_DIR/exec.d. execd_preinstall() # ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts series = get_series() # OMG?! is build-essentials needed? ubuntu_packages = [ 'python3-pip', 'python3-setuptools', 'python3-yaml', 'python3-dev', 'python3-wheel', 'build-essential' ] # I'm not going to "yum group info "Development Tools" # omitting above madness centos_packages = [ 'python3-pip', 'python3-setuptools', 'python3-devel', 'python3-wheel' ] packages_needed = [] if 'centos' in series: packages_needed = centos_packages else: packages_needed = ubuntu_packages charm_dir = os.environ['JUJU_CHARM_DIR'] os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin') venv = os.path.abspath('../.venv') vbin = os.path.join(venv, 'bin') vpip = os.path.join(vbin, 'pip') vpy = os.path.join(vbin, 'python') hook_name = os.path.basename(sys.argv[0]) is_bootstrapped = os.path.exists('wheelhouse/.bootstrapped') is_charm_upgrade = hook_name == 'upgrade-charm' is_series_upgrade = hook_name == 'post-series-upgrade' post_upgrade = os.path.exists('wheelhouse/.upgrade') is_upgrade = not post_upgrade and (is_charm_upgrade or is_series_upgrade) if is_bootstrapped and not is_upgrade: # older subordinates might have downgraded charm-env, so we should # restore it if necessary install_or_update_charm_env() activate_venv() # the .upgrade file prevents us from getting stuck in a loop # when re-execing to activate the venv; at this point, we've # activated the venv, so it's safe to clear it if post_upgrade: os.unlink('wheelhouse/.upgrade') return if is_series_upgrade and os.path.exists(venv): # series upgrade should do a full clear of the venv, rather than just # updating it, to bring in updates to Python itself shutil.rmtree(venv) if is_upgrade: if os.path.exists('wheelhouse/.bootstrapped'): os.unlink('wheelhouse/.bootstrapped') open('wheelhouse/.upgrade', 'w').close() # bootstrap wheelhouse if os.path.exists('wheelhouse'): with open('/root/.pydistutils.cfg', 'w') as fp: # make sure that easy_install also only uses the wheelhouse # (see https://github.com/pypa/pip/issues/410) fp.writelines([ "[easy_install]\n", "allow_hosts = ''\n", "find_links = file://{}/wheelhouse/\n".format(charm_dir), ]) if 'centos' in series: yum_install(packages_needed) else: apt_install(packages_needed) from charms.layer import options cfg = options.get('basic') # include packages defined in layer.yaml if 'centos' in series: yum_install(cfg.get('packages', [])) else: apt_install(cfg.get('packages', [])) # if we're using a venv, set it up if cfg.get('use_venv'): if not os.path.exists(venv): series = get_series() if series in ('ubuntu12.04', 'precise', 'ubuntu14.04', 'trusty'): apt_install(['python-virtualenv']) elif 'centos' in series: yum_install(['python-virtualenv']) else: apt_install(['virtualenv']) cmd = ['virtualenv', '-ppython3', '--never-download', venv] if cfg.get('include_system_packages'): cmd.append('--system-site-packages') check_call(cmd) os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']]) pip = vpip else: pip = 'pip3' # save a copy of system pip to prevent `pip3 install -U pip` # from changing it if os.path.exists('/usr/bin/pip'): shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save') # need newer pip, to fix spurious Double Requirement error: # https://github.com/pypa/pip/issues/56 check_call( [pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', 'pip']) # per https://github.com/juju-solutions/layer-basic/issues/110 # this replaces the setuptools that was copied over from the system on # venv create with latest setuptools and adds setuptools_scm check_call([ pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', 'setuptools', 'setuptools-scm' ]) # install the rest of the wheelhouse deps check_call([ pip, 'install', '-U', '--ignore-installed', '--no-index', '-f', 'wheelhouse' ] + glob('wheelhouse/*')) # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') # install pyyaml for centos7, since, unlike the ubuntu image, the # default image for centos doesn't include pyyaml; see the discussion: # https://discourse.jujucharms.com/t/charms-for-centos-lets-begin if 'centos' in series: check_call([pip, 'install', '-U', 'pyyaml']) # install python packages from layer options if cfg.get('python_packages'): check_call([pip, 'install', '-U'] + cfg.get('python_packages')) if not cfg.get('use_venv'): # restore system pip to prevent `pip3 install -U pip` # from changing it if os.path.exists('/usr/bin/pip.save'): shutil.copy2('/usr/bin/pip.save', '/usr/bin/pip') os.remove('/usr/bin/pip.save') # setup wrappers to ensure envs are used for scripts install_or_update_charm_env() for wrapper in ('charms.reactive', 'charms.reactive.sh', 'chlp', 'layer_option'): src = os.path.join('/usr/local/sbin', 'charm-env') dst = os.path.join('/usr/local/sbin', wrapper) if not os.path.exists(dst): os.symlink(src, dst) if cfg.get('use_venv'): shutil.copy2('bin/layer_option', vbin) else: shutil.copy2('bin/layer_option', '/usr/local/bin/') # re-link the charm copy to the wrapper in case charms # call bin/layer_option directly (as was the old pattern) os.remove('bin/layer_option') os.symlink('/usr/local/sbin/layer_option', 'bin/layer_option') # flag us as having already bootstrapped so we don't do it again open('wheelhouse/.bootstrapped', 'w').close() # Ensure that the newly bootstrapped libs are available. # Note: this only seems to be an issue with namespace packages. # Non-namespace-package libs (e.g., charmhelpers) are available # without having to reload the interpreter. :/ reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0])
network_get, status_set, ) from charmhelpers.core import unitdata from charmhelpers.core.host import ( service_running, service_start, service_restart ) from charms.layer import options if options.get('basic', 'use_venv'): PIP = os.path.join('../.venv', 'bin', 'pip') else: PIP = 'pip3' ES_HOME_DIR = Path('/usr/share/elasticsearch') ES_DATA_DIR = Path('/srv/elasticsearch-data') ES_DEFAULT_FILE_PATH = Path('/etc/default/elasticsearch') ES_PATH_CONF = Path('/etc/elasticsearch') ES_YML_PATH = ES_PATH_CONF / 'elasticsearch.yml' ES_PLUGIN = ES_HOME_DIR / 'bin' / 'elasticsearch-plugin'
from subprocess import check_call from pathlib import Path from os import environ from charmhelpers.core.hookenv import log, application_name from charms.reactive import not_unless from charms.layer import options _cfg = options.get("venv") ENV_NAME = _cfg["env_name"] if _cfg["env_name"] else application_name() log("ENV_NAME: {}".format(ENV_NAME)) ENV_DIR = Path("/opt/juju_venvs") / ENV_NAME ENV_BIN = ENV_DIR / "bin" @not_unless("venv.active") def call_from_env(args): """ Run command with arguments from inside the venv. Wait for command to complete. If the return code was zero then return, otherwise raise CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute. """ cmd = " ".join(args) log("Running {} from venv".format(cmd)) check_call(". {}/activate; {}".format(ENV_BIN, cmd), shell=True) def pip_install(package):
from charms.layer import options from charms.reactive import ( set_flag, when, when_not, ) from charmhelpers.core.host import is_container from charms.layer import status import charms.apt ELASTIC_PKGS = options.get('elastic-base', 'elastic-pkgs') @when_not('elastic.pkgs.available') def check_elastic_pkg_layer_option(): if len(ELASTIC_PKGS) > 0: set_flag('elastic.pkgs.available') else: status.blocked('elastic-base layer option for elastic-pkgs not set.') return # Install/Init ops # We have java, and know what elastic pkg to install, so lets get to it @when('elastic.pkgs.available', 'apt.installed.openjdk-8-jre-headless')
def add_pip_to_venv(): cfg = options.get("venv") for package in cfg["packages"]: pip_install(package) set_flag("venv.ready")
def bootstrap_charm_deps(): """ Set up the base charm dependencies so that the reactive system can run. """ # execd must happen first, before any attempt to install packages or # access the network, because sites use this hook to do bespoke # configuration and install secrets so the rest of this bootstrap # and the charm itself can actually succeed. This call does nothing # unless the operator has created and populated $JUJU_CHARM_DIR/exec.d. execd_preinstall() # ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts series = get_series() # OMG?! is build-essentials needed? ubuntu_packages = [ 'python3-pip', 'python3-setuptools', 'python3-yaml', 'python3-dev', 'python3-wheel', 'build-essential' ] # I'm not going to "yum group info "Development Tools" # omitting above madness centos_packages = [ 'python3-pip', 'python3-setuptools', 'python3-devel', 'python3-wheel' ] packages_needed = [] if 'centos' in series: packages_needed = centos_packages else: packages_needed = ubuntu_packages charm_dir = os.environ['JUJU_CHARM_DIR'] os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin') venv = os.path.abspath('../.venv') vbin = os.path.join(venv, 'bin') vpip = os.path.join(vbin, 'pip') vpy = os.path.join(vbin, 'python') hook_name = os.path.basename(sys.argv[0]) is_bootstrapped = os.path.exists('wheelhouse/.bootstrapped') is_charm_upgrade = hook_name == 'upgrade-charm' is_series_upgrade = hook_name == 'post-series-upgrade' is_post_upgrade = os.path.exists('wheelhouse/.upgraded') is_upgrade = (not is_post_upgrade and (is_charm_upgrade or is_series_upgrade)) if is_bootstrapped and not is_upgrade: # older subordinates might have downgraded charm-env, so we should # restore it if necessary install_or_update_charm_env() activate_venv() # the .upgrade file prevents us from getting stuck in a loop # when re-execing to activate the venv; at this point, we've # activated the venv, so it's safe to clear it if is_post_upgrade: os.unlink('wheelhouse/.upgraded') return if os.path.exists(venv): try: # focal installs or upgrades prior to PR 160 could leave the venv # in a broken state which would prevent subsequent charm upgrades _load_installed_versions(vpip) except CalledProcessError: is_broken_venv = True else: is_broken_venv = False if is_upgrade or is_broken_venv: # All upgrades should do a full clear of the venv, rather than # just updating it, to bring in updates to Python itself shutil.rmtree(venv) if is_upgrade: if os.path.exists('wheelhouse/.bootstrapped'): os.unlink('wheelhouse/.bootstrapped') # bootstrap wheelhouse if os.path.exists('wheelhouse'): pre_eoan = series in ('ubuntu12.04', 'precise', 'ubuntu14.04', 'trusty', 'ubuntu16.04', 'xenial', 'ubuntu18.04', 'bionic') pydistutils_lines = [ "[easy_install]\n", "find_links = file://{}/wheelhouse/\n".format(charm_dir), "no_index=True\n", "index_url=\n", # deliberately nothing here; disables it. ] if pre_eoan: pydistutils_lines.append("allow_hosts = ''\n") with open('/root/.pydistutils.cfg', 'w') as fp: # make sure that easy_install also only uses the wheelhouse # (see https://github.com/pypa/pip/issues/410) fp.writelines(pydistutils_lines) if 'centos' in series: yum_install(packages_needed) else: apt_install(packages_needed) from charms.layer import options cfg = options.get('basic') # include packages defined in layer.yaml if 'centos' in series: yum_install(cfg.get('packages', [])) else: apt_install(cfg.get('packages', [])) # if we're using a venv, set it up if cfg.get('use_venv'): if not os.path.exists(venv): series = get_series() if series in ('ubuntu12.04', 'precise', 'ubuntu14.04', 'trusty'): apt_install(['python-virtualenv']) elif 'centos' in series: yum_install(['python-virtualenv']) else: apt_install(['virtualenv']) cmd = ['virtualenv', '-ppython3', '--never-download', venv] if cfg.get('include_system_packages'): cmd.append('--system-site-packages') check_call(cmd, env=_get_subprocess_env()) os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']]) pip = vpip else: pip = 'pip3' # save a copy of system pip to prevent `pip3 install -U pip` # from changing it if os.path.exists('/usr/bin/pip'): shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save') pre_install_pkgs = ['pip', 'setuptools', 'setuptools-scm'] # we bundle these packages to work around bugs in older versions (such # as https://github.com/pypa/pip/issues/56), but if the system already # provided a newer version, downgrading it can cause other problems _update_if_newer(pip, pre_install_pkgs) # install the rest of the wheelhouse deps (extract the pkg names into # a set so that we can ignore the pre-install packages and let pip # choose the best version in case there are multiple from layer # conflicts) pkgs = _load_wheelhouse_versions().keys() - set(pre_install_pkgs) reinstall_flag = '--force-reinstall' if not cfg.get('use_venv', True) and pre_eoan: reinstall_flag = '--ignore-installed' check_call([ pip, 'install', '-U', reinstall_flag, '--no-index', '--no-cache-dir', '-f', 'wheelhouse' ] + list(pkgs), env=_get_subprocess_env()) # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') # install pyyaml for centos7, since, unlike the ubuntu image, the # default image for centos doesn't include pyyaml; see the discussion: # https://discourse.jujucharms.com/t/charms-for-centos-lets-begin if 'centos' in series: check_call([pip, 'install', '-U', 'pyyaml'], env=_get_subprocess_env()) # install python packages from layer options if cfg.get('python_packages'): check_call([pip, 'install', '-U'] + cfg.get('python_packages'), env=_get_subprocess_env()) if not cfg.get('use_venv'): # restore system pip to prevent `pip3 install -U pip` # from changing it if os.path.exists('/usr/bin/pip.save'): shutil.copy2('/usr/bin/pip.save', '/usr/bin/pip') os.remove('/usr/bin/pip.save') # setup wrappers to ensure envs are used for scripts install_or_update_charm_env() for wrapper in ('charms.reactive', 'charms.reactive.sh', 'chlp', 'layer_option'): src = os.path.join('/usr/local/sbin', 'charm-env') dst = os.path.join('/usr/local/sbin', wrapper) if not os.path.exists(dst): os.symlink(src, dst) if cfg.get('use_venv'): shutil.copy2('bin/layer_option', vbin) else: shutil.copy2('bin/layer_option', '/usr/local/bin/') # re-link the charm copy to the wrapper in case charms # call bin/layer_option directly (as was the old pattern) os.remove('bin/layer_option') os.symlink('/usr/local/sbin/layer_option', 'bin/layer_option') # flag us as having already bootstrapped so we don't do it again open('wheelhouse/.bootstrapped', 'w').close() if is_upgrade: # flag us as having already upgraded so we don't do it again open('wheelhouse/.upgraded', 'w').close() # Ensure that the newly bootstrapped libs are available. # Note: this only seems to be an issue with namespace packages. # Non-namespace-package libs (e.g., charmhelpers) are available # without having to reload the interpreter. :/ reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0])