def downed_migrate(prev_version):
"""Migration commands to be executed without any running services"""
if prev_version < StrictVersion('0.2.0'):
# Breaking changes: Influx downsampling model overhaul
# Old data is completely incompatible
utils.select(
'Upgrading to version >=0.2.0 requires a complete reset of your history data. '
+ "We'll be deleting it now")
sh('sudo rm -rf ./influxdb')
if prev_version < StrictVersion('0.3.0'):
# Splitting compose configuration between docker-compose and docker-compose.shared.yml
# Version pinning (0.2.2) will happen automatically
utils.info('Moving system services to docker-compose.shared.yml...')
config = utils.read_compose()
sys_names = [
'mdns',
'eventbus',
'influx',
'datastore',
'history',
'ui',
'traefik',
]
usr_config = {
'version': config['version'],
'services': {
key: svc
for (key, svc) in config['services'].items()
if key not in sys_names
}
}
utils.write_compose(usr_config)
if prev_version < StrictVersion('0.6.0'):
# The datastore service is gone
# Older services may still rely on it
utils.info('Removing `depends_on` fields from docker-compose.yml...')
config = utils.read_compose()
for svc in config['services'].values():
with suppress(KeyError):
del svc['depends_on']
utils.write_compose(config)
# Init dir. It will be filled during upped_migrate
utils.info('Creating redis/ dir...')
sh('mkdir -p redis/')
utils.info('Checking .env variables...')
for (key, default_value) in const.ENV_DEFAULTS.items():
current_value = utils.getenv(key)
if current_value is None:
utils.setenv(key, default_value)
def ports(http, https):
"""Update used ports"""
utils.check_config()
utils.confirm_mode()
cfg = {
const.HTTP_PORT_KEY: http,
const.HTTPS_PORT_KEY: https,
}
utils.info('Writing port settings to .env...')
for key, val in cfg.items():
utils.setenv(key, val)
def downed_migrate(prev_version):
"""Migration commands to be executed without any running services"""
if prev_version < StrictVersion('0.2.0'):
# Breaking changes: Influx downsampling model overhaul
# Old data is completely incompatible
utils.select(
'Upgrading to version >=0.2.0 requires a complete reset of your history data. '
+ "We'll be deleting it now")
sh('sudo rm -rf ./influxdb')
if prev_version < StrictVersion('0.3.0'):
# Splitting compose configuration between docker-compose and docker-compose.shared.yml
# Version pinning (0.2.2) will happen automatically
utils.info('Moving system services to docker-compose.shared.yml')
config = utils.read_compose()
sys_names = [
'mdns',
'eventbus',
'influx',
'datastore',
'history',
'ui',
'traefik',
]
usr_config = {
'version': config['version'],
'services': {
key: svc
for (key, svc) in config['services'].items()
if key not in sys_names
}
}
utils.write_compose(usr_config)
utils.info('Writing env values for all variables')
for key in [
const.COMPOSE_FILES_KEY,
const.RELEASE_KEY,
const.HTTP_PORT_KEY,
const.HTTPS_PORT_KEY,
const.MDNS_PORT_KEY,
]:
utils.setenv(key, utils.getenv(key, const.ENV_DEFAULTS[key]))
def setup(port_check):
"""Run first-time setup in Brewblox directory.
Run after brewblox-ctl install, in the newly created Brewblox directory.
This will create all required configuration files for your system.
You can safely use this command to partially reset your system.
Before making any changes, it will check for existing files,
and prompt if any are found. It will do so separately for docker-compose,
datastore, history, and gateway files.
Choose to skip any, and the others will still be created and configured.
\b
Steps:
- Check whether files already exist.
- Set .env values.
- Create docker-compose configuration files. (Optional)
- Pull docker images.
- Create datastore (CouchDB) directory. (Optional)
- Create history (InfluxDB) directory. (Optional)
- Create gateway (Traefik) directory. (Optional)
- Create SSL certificates. (Optional)
- Start and configure services. (Optional)
- Stop all services.
- Set version number in .env.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
datastore_url = utils.datastore_url()
history_url = utils.history_url()
upped_services = ['traefik', 'influx', 'history']
preset_modules = ['services', 'dashboards', 'dashboard-items']
if port_check:
check_ports()
skip_compose = \
utils.path_exists('./docker-compose.yml') \
and utils.confirm('This directory already contains a docker-compose.yml file. ' +
'Do you want to keep it?')
skip_datastore = \
utils.path_exists('./couchdb/') \
and utils.confirm('This directory already contains Couchdb datastore files. ' +
'Do you want to keep them?')
skip_history = \
utils.path_exists('./influxdb/') \
and utils.confirm('This directory already contains Influx history files. ' +
'Do you want to keep them?')
skip_gateway = \
utils.path_exists('./traefik/') \
and utils.confirm('This directory already contains Traefik gateway files. ' +
'Do you want to keep them?')
utils.info('Setting .env values...')
for key, default_val in const.ENV_DEFAULTS.items():
utils.setenv(key, utils.getenv(key, default_val))
if not skip_compose:
utils.info('Copying configuration...')
sh('cp -f {}/* ./'.format(const.CONFIG_DIR))
# Stop and pull after we're sure we have a compose file
utils.info('Stopping services...')
sh('{}docker-compose down --remove-orphans'.format(sudo))
utils.info('Pulling docker images...')
sh('{}docker-compose pull'.format(sudo))
if not skip_datastore:
utils.info('Creating datastore directory...')
upped_services.append('datastore')
sh('sudo rm -rf ./couchdb/; mkdir ./couchdb/')
if not skip_history:
utils.info('Creating history directory...')
sh('sudo rm -rf ./influxdb/; mkdir ./influxdb/')
if not skip_gateway:
utils.info('Creating gateway directory...')
sh('sudo rm -rf ./traefik/; mkdir ./traefik/')
utils.info('Creating SSL certificate...')
sh('{}docker run --rm -v "$(pwd)"/traefik/:/certs/ '.format(sudo) +
'brewblox/omgwtfssl:{}'.format(utils.docker_tag()))
sh('sudo chmod 644 traefik/brewblox.crt')
sh('sudo chmod 600 traefik/brewblox.key')
# Bring images online that we will send configuration
utils.info('Starting configured services...')
sh('{}docker-compose up -d --remove-orphans {}'.format(
sudo, ' '.join(upped_services)))
if not skip_datastore:
# Generic datastore setup
utils.info('Configuring datastore settings...')
sh('{} http wait {}'.format(const.CLI, datastore_url))
sh('{} http put {}/_users'.format(const.CLI, datastore_url))
sh('{} http put {}/_replicator'.format(const.CLI, datastore_url))
sh('{} http put {}/_global_changes'.format(const.CLI, datastore_url))
sh('{} http put {}/{}'.format(const.CLI, datastore_url,
const.UI_DATABASE))
# Load presets
utils.info('Loading preset data...')
for mod in preset_modules:
sh('{} http post {}/{}/_bulk_docs -f {}/{}.json'.format(
const.CLI, datastore_url, const.UI_DATABASE, const.PRESETS_DIR,
mod))
# Always setup history
utils.info('Configuring history settings...')
sh('{} http wait {}/ping'.format(const.CLI, history_url))
sh('{} http post {}/query/configure'.format(const.CLI, history_url))
# Setup is done - leave system in stable state
utils.info('Stopping services...')
sh('{}docker-compose down'.format(utils.optsudo()))
# Setup is complete and ok - now set CFG version
utils.setenv(const.CFG_VERSION_KEY, const.CURRENT_VERSION)
utils.info('All done!')
def update(ctx, update_ctl, update_ctl_done, pull, avahi_config, migrate,
prune, from_version):
"""Download and apply updates.
This is the one-stop-shop for updating your Brewblox install.
You can use any of the options to fine-tune the update by enabling or disabling subroutines.
By default, all options are enabled.
--update-ctl/--no-update-ctl determines whether it download new versions
of brewblox-ctl and brewblox-ctl lib. If this flag is set, update will download the new version
and then restart itself. This way, the migrate is done with the latest version of brewblox-ctl.
If you're using dry run mode, you'll notice the hidden option --update-ctl-done.
You can use it to watch the rest of the update: it\'s a flag to avoid endless loops.
--pull/--no-pull governs whether new docker images are pulled.
This is useful if any of your services is using a local image (not from Docker Hub).
--avahi-config/--no-avahi-config. Check avahi-daemon configuration.
This is required for TCP discovery of Spark controllers.
--migrate/--no-migrate. Updates regularly require changes to configuration.
To do this, services are stopped. If the update only requires pulling docker images,
you can disable migration to avoid the docker-compose down/up.
--prune/--no-prune (prompts if not set). Updates to docker images can leave unused old versions
on your system. These can be pruned to free up disk space.
Do note that this includes all images on your system, not just those created by Brewblox.
\b
Steps:
- Update brewblox-ctl and extensions.
- Restart update command to run with updated brewblox-ctl.
- Pull docker images.
- Stop services.
- Migrate configuration files.
- Copy docker-compose.shared.yml from defaults.
- Start services.
- Migrate service configuration.
- Write version number to .env file.
- Prune unused images.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
prev_version = StrictVersion(from_version)
if prev_version.version == (0, 0, 0):
click.echo(
'This configuration was never set up. Please run brewblox-ctl setup first'
)
raise SystemExit(1)
if prev_version > StrictVersion(const.CURRENT_VERSION):
click.echo(
'Your system is running a version newer than the selected release. '
+ 'This may be due to switching release tracks.' +
'You can use the --from-version flag if you know what you are doing.'
)
raise SystemExit(1)
if Path.home().name != 'root' and Path.home().exists() \
and Path('/usr/local/bin/brewblox-ctl').exists(): # pragma: no cover
utils.warn('brewblox-ctl appears to have been installed using sudo.')
if utils.confirm('Do you want to fix this now?'):
sh('sudo {} -m pip uninstall -y brewblox-ctl docker-compose'.
format(const.PY),
check=False)
utils.pip_install('brewblox-ctl') # docker-compose is a dependency
# Debian stretch still has the bug where ~/.local/bin is not included in $PATH
if '.local/bin' not in utils.getenv('PATH'):
sh('echo \'export PATH="$HOME/.local/bin:$PATH"\' >> ~/.bashrc'
)
utils.info(
'Please run "exec $SHELL --login" to apply the changes to $PATH'
)
return
if update_ctl and not update_ctl_done:
utils.info('Updating brewblox-ctl...')
utils.pip_install('brewblox-ctl')
utils.info('Updating brewblox-ctl libs...')
utils.load_ctl_lib()
# Restart ctl - we just replaced the source code
sh(' '.join([
const.PY, *const.ARGS, '--update-ctl-done',
'--prune' if prune else '--no-prune'
]))
return
if avahi_config:
utils.update_avahi_config()
if migrate:
# Everything except downed_migrate can be done with running services
utils.info('Stopping services...')
sh('{}docker-compose down --remove-orphans'.format(sudo))
utils.info('Migrating configuration files...')
apply_config()
downed_migrate(prev_version)
else:
utils.info('Updating configuration files...')
apply_config()
if pull:
utils.info('Pulling docker images...')
sh('{}docker-compose pull'.format(sudo))
utils.info('Starting services...')
sh('{}docker-compose up -d --remove-orphans'.format(sudo))
if migrate:
utils.info('Migrating service configuration...')
upped_migrate(prev_version)
utils.info('Updating version number to {}...'.format(
const.CURRENT_VERSION))
utils.setenv(const.CFG_VERSION_KEY, const.CURRENT_VERSION)
if prune:
utils.info('Pruning unused images...')
sh('{}docker image prune -f'.format(sudo))
utils.info('Pruning unused volumes...')
sh('{}docker volume prune -f'.format(sudo))
def setup(ctx, avahi_config, pull, port_check):
"""Run first-time setup in Brewblox directory.
Run after brewblox-ctl install, in the newly created Brewblox directory.
This will create all required configuration files for your system.
You can safely use this command to partially reset your system.
Before making any changes, it will check for existing files,
and prompt if any are found. It will do so separately for docker-compose,
datastore, history, and gateway files.
Choose to skip any, and the others will still be created and configured.
\b
Steps:
- Check whether files already exist.
- Set .env values.
- Update avahi-daemon config. (Optional)
- Create docker-compose configuration files. (Optional)
- Pull docker images. (Optional)
- Create datastore (Redis) directory. (Optional)
- Create history (Victoria) directory. (Optional)
- Create gateway (Traefik) directory. (Optional)
- Create SSL certificates. (Optional)
- Start and configure services. (Optional)
- Stop all services.
- Set version number in .env.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
if port_check:
check_ports()
skip_compose = \
utils.path_exists('./docker-compose.yml') \
and utils.confirm('This directory already contains a docker-compose.yml file. ' +
'Do you want to keep it?')
skip_datastore = \
utils.path_exists('./redis/') \
and utils.confirm('This directory already contains Redis datastore files. ' +
'Do you want to keep them?')
skip_history = \
utils.path_exists('./victoria/') \
and utils.confirm('This directory already contains Victoria history files. ' +
'Do you want to keep them?')
skip_gateway = \
utils.path_exists('./traefik/') \
and utils.confirm('This directory already contains Traefik gateway files. ' +
'Do you want to keep them?')
skip_eventbus = \
utils.path_exists('./mosquitto/') \
and utils.confirm('This directory already contains Mosquitto config files. ' +
'Do you want to keep them?')
utils.info('Setting .env values...')
for key, default_val in const.ENV_DEFAULTS.items():
utils.setenv(key, utils.getenv(key, default_val))
if avahi_config:
utils.update_avahi_config()
utils.info('Copying docker-compose.shared.yml...')
sh(f'cp -f {const.CONFIG_DIR}/docker-compose.shared.yml ./')
if not skip_compose:
utils.info('Copying docker-compose.yml...')
sh(f'cp -f {const.CONFIG_DIR}/docker-compose.yml ./')
# Stop and pull after we're sure we have a compose file
utils.info('Stopping services...')
sh(f'{sudo}docker-compose down')
if pull:
utils.info('Pulling docker images...')
sh(f'{sudo}docker-compose pull')
if not skip_datastore:
utils.info('Creating datastore directory...')
sh('sudo rm -rf ./redis/; mkdir ./redis/')
if not skip_history:
utils.info('Creating history directory...')
sh('sudo rm -rf ./victoria/; mkdir ./victoria/')
if not skip_gateway:
utils.info('Creating gateway directory...')
sh('sudo rm -rf ./traefik/; mkdir ./traefik/')
utils.info('Creating SSL certificate...')
ctx.invoke(makecert)
if not skip_eventbus:
utils.info('Creating mosquitto config directory...')
sh('sudo rm -rf ./mosquitto/; mkdir ./mosquitto/')
# Always copy cert config to traefik dir
sh(f'cp -f {const.CONFIG_DIR}/traefik-cert.yaml ./traefik/')
# Setup is complete and ok - now set CFG version
utils.setenv(const.CFG_VERSION_KEY, const.CURRENT_VERSION)
utils.info('All done!')