def activate_venv():
"""
Activate the venv if enabled in ``layer.yaml``.
This is handled automatically for normal hooks, but actions might
need to invoke this manually, using something like:
# Load modules from $JUJU_CHARM_DIR/lib
import sys
sys.path.append('lib')
from charms.layer.basic import activate_venv
activate_venv()
This will ensure that modules installed in the charm's
virtual environment are available to the action.
"""
from charms.layer import options
venv = os.path.abspath('../.venv')
vbin = os.path.join(venv, 'bin')
vpy = os.path.join(vbin, 'python')
use_venv = options.get('basic', 'use_venv')
if use_venv and '.venv' not in sys.executable:
# activate the venv
os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']])
reload_interpreter(vpy)
layer.patch_options_interface()
layer.import_layer_libs()
def reconfigure():
cfg = options.get("venv")
if "packages" in cfg:
if helpers.data_changed("venv-packages", cfg["packages"]):
clear_flag("venv.ready")
if helpers.data_changed("venv-name", ENV_NAME):
clear_flag("venv.active")
def configure(opts={}):
layer_opts = options.get('logrotate')
context = {}
context['options'] = {
**layer_opts,
**opts
} #merge options, param opts overwrite
target = LOGROTATE_DIR + service_name() + '.conf'
if context['options']:
log("Rendering {} with {}".format(target, context))
render(source=LOGROTATE_TEMPLATE, target=target, context=context)
def bootstrap_charm_deps():
"""
Set up the base charm dependencies so that the reactive system can run.
"""
# execd must happen first, before any attempt to install packages or
# access the network, because sites use this hook to do bespoke
# configuration and install secrets so the rest of this bootstrap
# and the charm itself can actually succeed. This call does nothing
# unless the operator has created and populated $JUJU_CHARM_DIR/exec.d.
execd_preinstall()
# ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts
series = get_series()
# OMG?! is build-essentials needed?
ubuntu_packages = [
'python3-pip', 'python3-setuptools', 'python3-yaml', 'python3-dev',
'python3-wheel', 'build-essential'
]
# I'm not going to "yum group info "Development Tools"
# omitting above madness
centos_packages = [
'python3-pip', 'python3-setuptools', 'python3-devel', 'python3-wheel'
]
packages_needed = []
if 'centos' in series:
packages_needed = centos_packages
else:
packages_needed = ubuntu_packages
charm_dir = os.environ['JUJU_CHARM_DIR']
os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin')
venv = os.path.abspath('../.venv')
vbin = os.path.join(venv, 'bin')
vpip = os.path.join(vbin, 'pip')
vpy = os.path.join(vbin, 'python')
hook_name = os.path.basename(sys.argv[0])
is_bootstrapped = os.path.exists('wheelhouse/.bootstrapped')
is_charm_upgrade = hook_name == 'upgrade-charm'
is_series_upgrade = hook_name == 'post-series-upgrade'
post_upgrade = os.path.exists('wheelhouse/.upgrade')
is_upgrade = not post_upgrade and (is_charm_upgrade or is_series_upgrade)
if is_bootstrapped and not is_upgrade:
# older subordinates might have downgraded charm-env, so we should
# restore it if necessary
install_or_update_charm_env()
activate_venv()
# the .upgrade file prevents us from getting stuck in a loop
# when re-execing to activate the venv; at this point, we've
# activated the venv, so it's safe to clear it
if post_upgrade:
os.unlink('wheelhouse/.upgrade')
return
if is_series_upgrade and os.path.exists(venv):
# series upgrade should do a full clear of the venv, rather than just
# updating it, to bring in updates to Python itself
shutil.rmtree(venv)
if is_upgrade:
if os.path.exists('wheelhouse/.bootstrapped'):
os.unlink('wheelhouse/.bootstrapped')
open('wheelhouse/.upgrade', 'w').close()
# bootstrap wheelhouse
if os.path.exists('wheelhouse'):
with open('/root/.pydistutils.cfg', 'w') as fp:
# make sure that easy_install also only uses the wheelhouse
# (see https://github.com/pypa/pip/issues/410)
fp.writelines([
"[easy_install]\n",
"allow_hosts = ''\n",
"find_links = file://{}/wheelhouse/\n".format(charm_dir),
])
if 'centos' in series:
yum_install(packages_needed)
else:
apt_install(packages_needed)
from charms.layer import options
cfg = options.get('basic')
# include packages defined in layer.yaml
if 'centos' in series:
yum_install(cfg.get('packages', []))
else:
apt_install(cfg.get('packages', []))
# if we're using a venv, set it up
if cfg.get('use_venv'):
if not os.path.exists(venv):
series = get_series()
if series in ('ubuntu12.04', 'precise', 'ubuntu14.04',
'trusty'):
apt_install(['python-virtualenv'])
elif 'centos' in series:
yum_install(['python-virtualenv'])
else:
apt_install(['virtualenv'])
cmd = ['virtualenv', '-ppython3', '--never-download', venv]
if cfg.get('include_system_packages'):
cmd.append('--system-site-packages')
check_call(cmd)
os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']])
pip = vpip
else:
pip = 'pip3'
# save a copy of system pip to prevent `pip3 install -U pip`
# from changing it
if os.path.exists('/usr/bin/pip'):
shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save')
# need newer pip, to fix spurious Double Requirement error:
# https://github.com/pypa/pip/issues/56
check_call(
[pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', 'pip'])
# per https://github.com/juju-solutions/layer-basic/issues/110
# this replaces the setuptools that was copied over from the system on
# venv create with latest setuptools and adds setuptools_scm
check_call([
pip, 'install', '-U', '--no-index', '-f', 'wheelhouse',
'setuptools', 'setuptools-scm'
])
# install the rest of the wheelhouse deps
check_call([
pip, 'install', '-U', '--ignore-installed', '--no-index', '-f',
'wheelhouse'
] + glob('wheelhouse/*'))
# re-enable installation from pypi
os.remove('/root/.pydistutils.cfg')
# install pyyaml for centos7, since, unlike the ubuntu image, the
# default image for centos doesn't include pyyaml; see the discussion:
# https://discourse.jujucharms.com/t/charms-for-centos-lets-begin
if 'centos' in series:
check_call([pip, 'install', '-U', 'pyyaml'])
# install python packages from layer options
if cfg.get('python_packages'):
check_call([pip, 'install', '-U'] + cfg.get('python_packages'))
if not cfg.get('use_venv'):
# restore system pip to prevent `pip3 install -U pip`
# from changing it
if os.path.exists('/usr/bin/pip.save'):
shutil.copy2('/usr/bin/pip.save', '/usr/bin/pip')
os.remove('/usr/bin/pip.save')
# setup wrappers to ensure envs are used for scripts
install_or_update_charm_env()
for wrapper in ('charms.reactive', 'charms.reactive.sh', 'chlp',
'layer_option'):
src = os.path.join('/usr/local/sbin', 'charm-env')
dst = os.path.join('/usr/local/sbin', wrapper)
if not os.path.exists(dst):
os.symlink(src, dst)
if cfg.get('use_venv'):
shutil.copy2('bin/layer_option', vbin)
else:
shutil.copy2('bin/layer_option', '/usr/local/bin/')
# re-link the charm copy to the wrapper in case charms
# call bin/layer_option directly (as was the old pattern)
os.remove('bin/layer_option')
os.symlink('/usr/local/sbin/layer_option', 'bin/layer_option')
# flag us as having already bootstrapped so we don't do it again
open('wheelhouse/.bootstrapped', 'w').close()
# Ensure that the newly bootstrapped libs are available.
# Note: this only seems to be an issue with namespace packages.
# Non-namespace-package libs (e.g., charmhelpers) are available
# without having to reload the interpreter. :/
reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0])
network_get,
status_set,
)
from charmhelpers.core import unitdata
from charmhelpers.core.host import (
service_running,
service_start,
service_restart
)
from charms.layer import options
if options.get('basic', 'use_venv'):
PIP = os.path.join('../.venv', 'bin', 'pip')
else:
PIP = 'pip3'
ES_HOME_DIR = Path('/usr/share/elasticsearch')
ES_DATA_DIR = Path('/srv/elasticsearch-data')
ES_DEFAULT_FILE_PATH = Path('/etc/default/elasticsearch')
ES_PATH_CONF = Path('/etc/elasticsearch')
ES_YML_PATH = ES_PATH_CONF / 'elasticsearch.yml'
ES_PLUGIN = ES_HOME_DIR / 'bin' / 'elasticsearch-plugin'
from subprocess import check_call
from pathlib import Path
from os import environ
from charmhelpers.core.hookenv import log, application_name
from charms.reactive import not_unless
from charms.layer import options
_cfg = options.get("venv")
ENV_NAME = _cfg["env_name"] if _cfg["env_name"] else application_name()
log("ENV_NAME: {}".format(ENV_NAME))
ENV_DIR = Path("/opt/juju_venvs") / ENV_NAME
ENV_BIN = ENV_DIR / "bin"
@not_unless("venv.active")
def call_from_env(args):
"""
Run command with arguments from inside the venv. Wait for command
to complete. If the return code was zero then return, otherwise
raise CalledProcessError. The CalledProcessError object will have
the return code in the returncode attribute.
"""
cmd = " ".join(args)
log("Running {} from venv".format(cmd))
check_call(". {}/activate; {}".format(ENV_BIN, cmd), shell=True)
def pip_install(package):
from charms.layer import options
from charms.reactive import (
set_flag,
when,
when_not,
)
from charmhelpers.core.host import is_container
from charms.layer import status
import charms.apt
ELASTIC_PKGS = options.get('elastic-base', 'elastic-pkgs')
@when_not('elastic.pkgs.available')
def check_elastic_pkg_layer_option():
if len(ELASTIC_PKGS) > 0:
set_flag('elastic.pkgs.available')
else:
status.blocked('elastic-base layer option for elastic-pkgs not set.')
return
# Install/Init ops
# We have java, and know what elastic pkg to install, so lets get to it
@when('elastic.pkgs.available',
'apt.installed.openjdk-8-jre-headless')
def add_pip_to_venv():
cfg = options.get("venv")
for package in cfg["packages"]:
pip_install(package)
set_flag("venv.ready")
def bootstrap_charm_deps():
"""
Set up the base charm dependencies so that the reactive system can run.
"""
# execd must happen first, before any attempt to install packages or
# access the network, because sites use this hook to do bespoke
# configuration and install secrets so the rest of this bootstrap
# and the charm itself can actually succeed. This call does nothing
# unless the operator has created and populated $JUJU_CHARM_DIR/exec.d.
execd_preinstall()
# ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts
series = get_series()
# OMG?! is build-essentials needed?
ubuntu_packages = [
'python3-pip', 'python3-setuptools', 'python3-yaml', 'python3-dev',
'python3-wheel', 'build-essential'
]
# I'm not going to "yum group info "Development Tools"
# omitting above madness
centos_packages = [
'python3-pip', 'python3-setuptools', 'python3-devel', 'python3-wheel'
]
packages_needed = []
if 'centos' in series:
packages_needed = centos_packages
else:
packages_needed = ubuntu_packages
charm_dir = os.environ['JUJU_CHARM_DIR']
os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin')
venv = os.path.abspath('../.venv')
vbin = os.path.join(venv, 'bin')
vpip = os.path.join(vbin, 'pip')
vpy = os.path.join(vbin, 'python')
hook_name = os.path.basename(sys.argv[0])
is_bootstrapped = os.path.exists('wheelhouse/.bootstrapped')
is_charm_upgrade = hook_name == 'upgrade-charm'
is_series_upgrade = hook_name == 'post-series-upgrade'
is_post_upgrade = os.path.exists('wheelhouse/.upgraded')
is_upgrade = (not is_post_upgrade
and (is_charm_upgrade or is_series_upgrade))
if is_bootstrapped and not is_upgrade:
# older subordinates might have downgraded charm-env, so we should
# restore it if necessary
install_or_update_charm_env()
activate_venv()
# the .upgrade file prevents us from getting stuck in a loop
# when re-execing to activate the venv; at this point, we've
# activated the venv, so it's safe to clear it
if is_post_upgrade:
os.unlink('wheelhouse/.upgraded')
return
if os.path.exists(venv):
try:
# focal installs or upgrades prior to PR 160 could leave the venv
# in a broken state which would prevent subsequent charm upgrades
_load_installed_versions(vpip)
except CalledProcessError:
is_broken_venv = True
else:
is_broken_venv = False
if is_upgrade or is_broken_venv:
# All upgrades should do a full clear of the venv, rather than
# just updating it, to bring in updates to Python itself
shutil.rmtree(venv)
if is_upgrade:
if os.path.exists('wheelhouse/.bootstrapped'):
os.unlink('wheelhouse/.bootstrapped')
# bootstrap wheelhouse
if os.path.exists('wheelhouse'):
pre_eoan = series in ('ubuntu12.04', 'precise', 'ubuntu14.04',
'trusty', 'ubuntu16.04', 'xenial', 'ubuntu18.04',
'bionic')
pydistutils_lines = [
"[easy_install]\n",
"find_links = file://{}/wheelhouse/\n".format(charm_dir),
"no_index=True\n",
"index_url=\n", # deliberately nothing here; disables it.
]
if pre_eoan:
pydistutils_lines.append("allow_hosts = ''\n")
with open('/root/.pydistutils.cfg', 'w') as fp:
# make sure that easy_install also only uses the wheelhouse
# (see https://github.com/pypa/pip/issues/410)
fp.writelines(pydistutils_lines)
if 'centos' in series:
yum_install(packages_needed)
else:
apt_install(packages_needed)
from charms.layer import options
cfg = options.get('basic')
# include packages defined in layer.yaml
if 'centos' in series:
yum_install(cfg.get('packages', []))
else:
apt_install(cfg.get('packages', []))
# if we're using a venv, set it up
if cfg.get('use_venv'):
if not os.path.exists(venv):
series = get_series()
if series in ('ubuntu12.04', 'precise', 'ubuntu14.04',
'trusty'):
apt_install(['python-virtualenv'])
elif 'centos' in series:
yum_install(['python-virtualenv'])
else:
apt_install(['virtualenv'])
cmd = ['virtualenv', '-ppython3', '--never-download', venv]
if cfg.get('include_system_packages'):
cmd.append('--system-site-packages')
check_call(cmd, env=_get_subprocess_env())
os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']])
pip = vpip
else:
pip = 'pip3'
# save a copy of system pip to prevent `pip3 install -U pip`
# from changing it
if os.path.exists('/usr/bin/pip'):
shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save')
pre_install_pkgs = ['pip', 'setuptools', 'setuptools-scm']
# we bundle these packages to work around bugs in older versions (such
# as https://github.com/pypa/pip/issues/56), but if the system already
# provided a newer version, downgrading it can cause other problems
_update_if_newer(pip, pre_install_pkgs)
# install the rest of the wheelhouse deps (extract the pkg names into
# a set so that we can ignore the pre-install packages and let pip
# choose the best version in case there are multiple from layer
# conflicts)
pkgs = _load_wheelhouse_versions().keys() - set(pre_install_pkgs)
reinstall_flag = '--force-reinstall'
if not cfg.get('use_venv', True) and pre_eoan:
reinstall_flag = '--ignore-installed'
check_call([
pip, 'install', '-U', reinstall_flag, '--no-index',
'--no-cache-dir', '-f', 'wheelhouse'
] + list(pkgs),
env=_get_subprocess_env())
# re-enable installation from pypi
os.remove('/root/.pydistutils.cfg')
# install pyyaml for centos7, since, unlike the ubuntu image, the
# default image for centos doesn't include pyyaml; see the discussion:
# https://discourse.jujucharms.com/t/charms-for-centos-lets-begin
if 'centos' in series:
check_call([pip, 'install', '-U', 'pyyaml'],
env=_get_subprocess_env())
# install python packages from layer options
if cfg.get('python_packages'):
check_call([pip, 'install', '-U'] + cfg.get('python_packages'),
env=_get_subprocess_env())
if not cfg.get('use_venv'):
# restore system pip to prevent `pip3 install -U pip`
# from changing it
if os.path.exists('/usr/bin/pip.save'):
shutil.copy2('/usr/bin/pip.save', '/usr/bin/pip')
os.remove('/usr/bin/pip.save')
# setup wrappers to ensure envs are used for scripts
install_or_update_charm_env()
for wrapper in ('charms.reactive', 'charms.reactive.sh', 'chlp',
'layer_option'):
src = os.path.join('/usr/local/sbin', 'charm-env')
dst = os.path.join('/usr/local/sbin', wrapper)
if not os.path.exists(dst):
os.symlink(src, dst)
if cfg.get('use_venv'):
shutil.copy2('bin/layer_option', vbin)
else:
shutil.copy2('bin/layer_option', '/usr/local/bin/')
# re-link the charm copy to the wrapper in case charms
# call bin/layer_option directly (as was the old pattern)
os.remove('bin/layer_option')
os.symlink('/usr/local/sbin/layer_option', 'bin/layer_option')
# flag us as having already bootstrapped so we don't do it again
open('wheelhouse/.bootstrapped', 'w').close()
if is_upgrade:
# flag us as having already upgraded so we don't do it again
open('wheelhouse/.upgraded', 'w').close()
# Ensure that the newly bootstrapped libs are available.
# Note: this only seems to be an issue with namespace packages.
# Non-namespace-package libs (e.g., charmhelpers) are available
# without having to reload the interpreter. :/
reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0])
