def add_node_red(force):
"""
Create a service for Node-RED.
"""
utils.check_config()
utils.confirm_mode()
name = 'node-red'
sudo = utils.optsudo()
host = utils.host_ip()
port = utils.getenv(const.HTTPS_PORT_KEY)
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/node-red:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'volumes': [
f'./{name}:/data',
]
}
sh(f'mkdir -p ./{name}')
if [getgid(), getuid()] != [1000, 1000]:
sh(f'sudo chown -R 1000:1000 ./{name}')
utils.write_compose(config)
click.echo(f'Added Node-RED service `{name}`.')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
click.echo(
f'Visit https://{host}:{port}/{name} in your browser to load the editor.'
)
def add_plaato(name, token, force):
"""
Create a service for the Plaato airlock.
This will periodically query the Plaato server for current state.
An authentication token is required.
See https://plaato.io/apps/help-center#!hc-auth-token on how to get one.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/brewblox-plaato:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'environment': {
'PLAATO_AUTH': token,
},
'command': f'--name={name}',
}
utils.write_compose(config)
click.echo(f'Added Plaato service `{name}`.')
click.echo(
'This service publishes history data, but does not have a UI component.'
)
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
def add_tilt(force):
"""
Create a service for the Tilt hydrometer.
The service listens for Bluetooth status updates from the Tilt,
and requires the host to have a Bluetooth receiver.
The empty ./tilt dir is created to hold calibration files.
"""
utils.check_config()
utils.confirm_mode()
name = 'tilt'
sudo = utils.optsudo()
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/brewblox-tilt:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'privileged': True,
'network_mode': 'host',
'volumes': [
f'./{name}:/share',
],
'labels': [
'traefik.enable=false',
],
}
sh(f'mkdir -p ./{name}')
utils.write_compose(config)
click.echo(f'Added Tilt service `{name}`.')
click.echo('It will automatically show up in the UI.\n')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
def save(save_compose, ignore_spark_error):
"""Create a backup of Brewblox settings.
A zip archive containing JSON/YAML files is created in the ./backup/ directory.
The archive name will include current date and time to ensure uniqueness.
The backup is not exported to any kind of remote/cloud storage.
To use this command in scripts, run it as `brewblox-ctl --quiet backup save`.
Its only output to stdout will be the absolute path to the created backup.
The command will fail if any of the Spark services could not be contacted.
As it does not make any destructive changes to configuration,
this command is not affected by --dry-run.
\b
Stored data:
- .env
- docker-compose.yml. (Optional)
- Datastore databases.
- Spark service blocks.
\b
NOT stored:
- History data.
"""
utils.check_config()
urllib3.disable_warnings()
file = 'backup/brewblox_backup_{}.zip'.format(
datetime.now().strftime('%Y%m%d_%H%M'))
with suppress(FileExistsError):
mkdir(path.abspath('backup/'))
url = utils.datastore_url()
utils.info('Waiting for the datastore...')
http.wait(url, info_updates=True)
resp = requests.get(url + '/_all_dbs', verify=False)
resp.raise_for_status()
dbs = [v for v in resp.json() if not v.startswith('_')]
config = utils.read_compose()
sparks = [
k for k, v in config['services'].items()
if v.get('image', '').startswith('brewblox/brewblox-devcon-spark')
]
zipf = zipfile.ZipFile(file, 'w', zipfile.ZIP_DEFLATED)
utils.info('Exporting .env')
zipf.write('.env')
if save_compose:
utils.info('Exporting docker-compose.yml')
zipf.write('docker-compose.yml')
utils.info('Exporting databases: {}'.format(', '.join(dbs)))
for db in dbs:
resp = requests.get('{}/{}/_all_docs'.format(url, db),
params={'include_docs': True},
verify=False)
resp.raise_for_status()
docs = [v['doc'] for v in resp.json()['rows']]
for d in docs:
del d['_rev']
zipf.writestr(db + '.datastore.json', json.dumps(docs))
for spark in sparks:
utils.info("Exporting Spark blocks from '{}'".format(spark))
resp = requests.get('{}/{}/export_objects'.format(
utils.host_url(), spark),
verify=False)
try:
resp.raise_for_status()
zipf.writestr(spark + '.spark.json', resp.text)
except Exception as ex:
if ignore_spark_error:
utils.info("Skipping '{}' due to error: {}".format(
spark, str(ex)))
else:
raise ex
zipf.close()
click.echo(path.abspath(file))
utils.info('Done!')
def load(archive, load_env, load_compose, load_datastore, load_spark, update):
"""Load and apply Brewblox settings backup.
This function uses files generated by `brewblox-ctl backup save` as input.
You can use the --load-XXXX options to partially load the backup.
This does not attempt to merge data: it will overwrite current docker-compose.yml,
datastore databases, and Spark blocks.
Blocks on Spark services not in the backup file will not be affected.
If dry-run is enabled, it will echo all configuration from the backup archive.
Steps:
- Write .env
- Read .env values
- Write docker-compose.yml, run `docker-compose up`.
- Write all datastore files found in backup.
- Write all Spark blocks found in backup.
- Run brewblox-ctl update
"""
utils.check_config()
utils.confirm_mode()
urllib3.disable_warnings()
sudo = utils.optsudo()
host_url = utils.host_url()
store_url = utils.datastore_url()
zipf = zipfile.ZipFile(archive, 'r', zipfile.ZIP_DEFLATED)
available = zipf.namelist()
datastore_files = [v for v in available if v.endswith('.datastore.json')]
spark_files = [v for v in available if v.endswith('.spark.json')]
if load_env and '.env' in available:
with NamedTemporaryFile('w') as tmp:
data = zipf.read('.env').decode()
utils.info('Writing .env')
utils.show_data(data)
tmp.write(data)
tmp.flush()
sh('cp -f {} .env'.format(tmp.name))
utils.info('Reading .env values')
load_dotenv(path.abspath('.env'))
if load_compose:
if 'docker-compose.yml' in available:
utils.info('Writing docker-compose.yml')
utils.write_compose(yaml.safe_load(
zipf.read('docker-compose.yml')))
sh('{} docker-compose up -d --remove-orphans'.format(sudo))
else:
utils.info('docker-compose.yml file not found in backup archive')
if load_datastore:
if datastore_files:
utils.info('Waiting for the datastore...')
sh('{} http wait {}'.format(const.CLI, store_url))
else:
utils.info('No datastore files found in backup archive')
for f in datastore_files:
db = f[:-len('.datastore.json')]
utils.info('Recreating database {}'.format(db))
sh('{} http delete {}/{} --allow-fail'.format(
const.CLI, store_url, db))
sh('{} http put {}/{}'.format(const.CLI, store_url, db))
utils.info('Writing database {}'.format(db))
with NamedTemporaryFile('w') as tmp:
data = {'docs': json.loads(zipf.read(f).decode())}
utils.show_data(data)
json.dump(data, tmp)
tmp.flush()
sh('{} http post {}/{}/_bulk_docs -f {}'.format(
const.CLI, store_url, db, tmp.name))
if load_spark:
sudo = utils.optsudo()
if not spark_files:
utils.info('No Spark files found in backup archive')
for f in spark_files:
spark = f[:-len('.spark.json')]
utils.info('Writing blocks to Spark service {}'.format(spark))
with NamedTemporaryFile('w') as tmp:
data = json.loads(zipf.read(f).decode())
utils.show_data(data)
json.dump(data, tmp)
tmp.flush()
sh('{} http post {}/{}/import_objects -f {}'.format(
const.CLI, host_url, spark, tmp.name))
sh('{} docker-compose restart {}'.format(sudo, spark))
zipf.close()
if update:
utils.info('Updating brewblox...')
sh('{} update'.format(const.CLI))
utils.info('Done!')
def update(ctx, update_ctl, update_ctl_done, pull, avahi_config, migrate,
prune, from_version):
"""Download and apply updates.
This is the one-stop-shop for updating your Brewblox install.
You can use any of the options to fine-tune the update by enabling or disabling subroutines.
By default, all options are enabled.
--update-ctl/--no-update-ctl determines whether it download new versions
of brewblox-ctl and brewblox-ctl lib. If this flag is set, update will download the new version
and then restart itself. This way, the migrate is done with the latest version of brewblox-ctl.
If you're using dry run mode, you'll notice the hidden option --update-ctl-done.
You can use it to watch the rest of the update: it\'s a flag to avoid endless loops.
--pull/--no-pull governs whether new docker images are pulled.
This is useful if any of your services is using a local image (not from Docker Hub).
--avahi-config/--no-avahi-config. Check avahi-daemon configuration.
This is required for TCP discovery of Spark controllers.
--migrate/--no-migrate. Updates regularly require changes to configuration.
To do this, services are stopped. If the update only requires pulling docker images,
you can disable migration to avoid the docker-compose down/up.
--prune/--no-prune (prompts if not set). Updates to docker images can leave unused old versions
on your system. These can be pruned to free up disk space.
Do note that this includes all images on your system, not just those created by Brewblox.
\b
Steps:
- Update brewblox-ctl and extensions.
- Restart update command to run with updated brewblox-ctl.
- Pull docker images.
- Stop services.
- Migrate configuration files.
- Copy docker-compose.shared.yml from defaults.
- Start services.
- Migrate service configuration.
- Write version number to .env file.
- Prune unused images.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
prev_version = StrictVersion(from_version)
if prev_version.version == (0, 0, 0):
click.echo(
'This configuration was never set up. Please run brewblox-ctl setup first'
)
raise SystemExit(1)
if prev_version > StrictVersion(const.CURRENT_VERSION):
click.echo(
'Your system is running a version newer than the selected release. '
+ 'This may be due to switching release tracks.' +
'You can use the --from-version flag if you know what you are doing.'
)
raise SystemExit(1)
if Path.home().name != 'root' and Path.home().exists() \
and Path('/usr/local/bin/brewblox-ctl').exists(): # pragma: no cover
utils.warn('brewblox-ctl appears to have been installed using sudo.')
if utils.confirm('Do you want to fix this now?'):
sh('sudo {} -m pip uninstall -y brewblox-ctl docker-compose'.
format(const.PY),
check=False)
utils.pip_install('brewblox-ctl') # docker-compose is a dependency
# Debian stretch still has the bug where ~/.local/bin is not included in $PATH
if '.local/bin' not in utils.getenv('PATH'):
sh('echo \'export PATH="$HOME/.local/bin:$PATH"\' >> ~/.bashrc'
)
utils.info(
'Please run "exec $SHELL --login" to apply the changes to $PATH'
)
return
if update_ctl and not update_ctl_done:
utils.info('Updating brewblox-ctl...')
utils.pip_install('brewblox-ctl')
utils.info('Updating brewblox-ctl libs...')
utils.load_ctl_lib()
# Restart ctl - we just replaced the source code
sh(' '.join([
const.PY, *const.ARGS, '--update-ctl-done',
'--prune' if prune else '--no-prune'
]))
return
if avahi_config:
utils.update_avahi_config()
if migrate:
# Everything except downed_migrate can be done with running services
utils.info('Stopping services...')
sh('{}docker-compose down --remove-orphans'.format(sudo))
utils.info('Migrating configuration files...')
apply_config()
downed_migrate(prev_version)
else:
utils.info('Updating configuration files...')
apply_config()
if pull:
utils.info('Pulling docker images...')
sh('{}docker-compose pull'.format(sudo))
utils.info('Starting services...')
sh('{}docker-compose up -d --remove-orphans'.format(sudo))
if migrate:
utils.info('Migrating service configuration...')
upped_migrate(prev_version)
utils.info('Updating version number to {}...'.format(
const.CURRENT_VERSION))
utils.setenv(const.CFG_VERSION_KEY, const.CURRENT_VERSION)
if prune:
utils.info('Pruning unused images...')
sh('{}docker image prune -f'.format(sudo))
utils.info('Pruning unused volumes...')
sh('{}docker volume prune -f'.format(sudo))
def update(update_ctl, update_ctl_done, pull, update_system, migrate, prune,
from_version):
"""Download and apply updates.
This is the one-stop-shop for updating your Brewblox install.
You can use any of the options to fine-tune the update by enabling or disabling subroutines.
By default, all options are enabled.
--update-ctl/--no-update-ctl: Whether to download and install new versions of
of brewblox-ctl and brewblox-ctl lib. If this flag is set, update will download the new version
and then restart itself. This way, the migrate is done with the latest version of brewblox-ctl.
If you're using dry run mode, you'll notice the hidden option --update-ctl-done.
You can use it to watch the rest of the update: it\'s a flag to avoid endless loops.
--pull/--no-pull. Whether to pull docker images.
This is useful if any of your services is using a local image (not from Docker Hub).
--update-system/--no-update-system determines whether
--migrate/--no-migrate. Updates regularly require changes to configuration.
Required changes are applied here.
--prune/--no-prune. Updates to docker images can leave unused old versions
on your system. These can be pruned to free up disk space.
This includes all images and volumes on your system, and not just those created by Brewblox.
\b
Steps:
- Check whether any system fixes must be applied.
- Update brewblox-ctl and brewblox-ctl libs.
- Stop services.
- Update Avahi config.
- Update system packages.
- Migrate configuration files.
- Pull Docker images.
- Prune unused Docker images and volumes.
- Start services.
- Migrate service configuration.
- Write version number to .env file.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
prev_version = StrictVersion(from_version)
check_version(prev_version)
if update_ctl and not update_ctl_done:
utils.info('Updating brewblox-ctl...')
utils.pip_install('brewblox-ctl')
utils.info('Updating brewblox-ctl lib...')
utils.load_ctl_lib()
# Restart ctl - we just replaced the source code
sh(' '.join([const.PY, *const.ARGS, '--update-ctl-done']))
return
utils.info('Stopping services...')
sh(f'{sudo}docker-compose down')
# Download and install the new brewblox-ctl
utils.info('Upgrading brewblox-ctl to the unified version...')
if utils.command_exists('apt') and update_system: # pragma: no branch
utils.info('Installing dependencies...')
sh('sudo apt update && sudo apt install -y python3-venv')
release = utils.getenv(const.RELEASE_KEY)
sh(f'wget -q -O ./brewblox-ctl.tar.gz https://brewblox.blob.core.windows.net/ctl/{release}/brewblox-ctl.tar.gz'
)
utils.info('Creating virtual env...')
sh(f'{const.PY} -m venv .venv')
utils.info('Installing packages...')
sh(' && '.join([
'. .venv/bin/activate',
'python3 -m pip install setuptools wheel pip',
'python3 -m pip install ./brewblox-ctl.tar.gz',
' '.join(['python3 -m brewblox_ctl',
*const.ARGS[1:]]), # Already have the --update-ctl-done arg
]))
def setup(ctx, avahi_config, pull, port_check):
"""Run first-time setup in Brewblox directory.
Run after brewblox-ctl install, in the newly created Brewblox directory.
This will create all required configuration files for your system.
You can safely use this command to partially reset your system.
Before making any changes, it will check for existing files,
and prompt if any are found. It will do so separately for docker-compose,
datastore, history, and gateway files.
Choose to skip any, and the others will still be created and configured.
\b
Steps:
- Check whether files already exist.
- Set .env values.
- Update avahi-daemon config. (Optional)
- Create docker-compose configuration files. (Optional)
- Pull docker images. (Optional)
- Create datastore (Redis) directory. (Optional)
- Create history (Victoria) directory. (Optional)
- Create gateway (Traefik) directory. (Optional)
- Create SSL certificates. (Optional)
- Start and configure services. (Optional)
- Stop all services.
- Set version number in .env.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
if port_check:
check_ports()
skip_compose = \
utils.path_exists('./docker-compose.yml') \
and utils.confirm('This directory already contains a docker-compose.yml file. ' +
'Do you want to keep it?')
skip_datastore = \
utils.path_exists('./redis/') \
and utils.confirm('This directory already contains Redis datastore files. ' +
'Do you want to keep them?')
skip_history = \
utils.path_exists('./victoria/') \
and utils.confirm('This directory already contains Victoria history files. ' +
'Do you want to keep them?')
skip_gateway = \
utils.path_exists('./traefik/') \
and utils.confirm('This directory already contains Traefik gateway files. ' +
'Do you want to keep them?')
skip_eventbus = \
utils.path_exists('./mosquitto/') \
and utils.confirm('This directory already contains Mosquitto config files. ' +
'Do you want to keep them?')
utils.info('Setting .env values...')
for key, default_val in const.ENV_DEFAULTS.items():
utils.setenv(key, utils.getenv(key, default_val))
if avahi_config:
utils.update_avahi_config()
utils.info('Copying docker-compose.shared.yml...')
sh(f'cp -f {const.CONFIG_DIR}/docker-compose.shared.yml ./')
if not skip_compose:
utils.info('Copying docker-compose.yml...')
sh(f'cp -f {const.CONFIG_DIR}/docker-compose.yml ./')
# Stop and pull after we're sure we have a compose file
utils.info('Stopping services...')
sh(f'{sudo}docker-compose down')
if pull:
utils.info('Pulling docker images...')
sh(f'{sudo}docker-compose pull')
if not skip_datastore:
utils.info('Creating datastore directory...')
sh('sudo rm -rf ./redis/; mkdir ./redis/')
if not skip_history:
utils.info('Creating history directory...')
sh('sudo rm -rf ./victoria/; mkdir ./victoria/')
if not skip_gateway:
utils.info('Creating gateway directory...')
sh('sudo rm -rf ./traefik/; mkdir ./traefik/')
utils.info('Creating SSL certificate...')
ctx.invoke(makecert)
if not skip_eventbus:
utils.info('Creating mosquitto config directory...')
sh('sudo rm -rf ./mosquitto/; mkdir ./mosquitto/')
# Always copy cert config to traefik dir
sh(f'cp -f {const.CONFIG_DIR}/traefik-cert.yaml ./traefik/')
# Setup is complete and ok - now set CFG version
utils.setenv(const.CFG_VERSION_KEY, const.CURRENT_VERSION)
utils.info('All done!')
