def add_node_red(force):
"""
Create a service for Node-RED.
"""
utils.check_config()
utils.confirm_mode()
name = 'node-red'
sudo = utils.optsudo()
host = utils.host_ip()
port = utils.getenv(const.HTTPS_PORT_KEY)
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/node-red:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'volumes': [{
'type': 'bind',
'source': f'./{name}',
'target': '/data',
}]
}
sh(f'mkdir -p ./{name}')
if [getgid(), getuid()] != [1000, 1000]:
sh(f'sudo chown -R 1000:1000 ./{name}')
utils.write_compose(config)
click.echo(f'Added Node-RED service `{name}`.')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
click.echo(f'Visit https://{host}:{port}/{name} in your browser to load the editor.')
def show(image, file):
"""Show all services of a specific type.
Use the --image flag to filter."""
utils.check_config()
services = utils.list_services(image, file)
click.echo('\n'.join(services), nl=bool(services))
def add_plaato(name, token, force):
"""
Create a service for the Plaato airlock.
This will periodically query the Plaato server for current state.
An authentication token is required.
See https://plaato.io/apps/help-center#!hc-auth-token on how to get one.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/brewblox-plaato:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'environment': {
'PLAATO_AUTH': token,
},
'command': f'--name={name}',
}
utils.write_compose(config)
click.echo(f'Added Plaato service `{name}`.')
click.echo('This service publishes history data, but does not have a UI component.')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
def list_env():
"""List all .env variables
This does not include other variables set in the current shell.
"""
utils.check_config()
for k, v in dotenv.dotenv_values('.env').items():
click.echo(f'{k} = {v}')
def set_env(key, value):
"""Set a .env variable.
The value will be added to the .env file. You can set new variables with this.
"""
utils.check_config()
utils.confirm_mode()
utils.setenv(key, value)
def skip_confirm(value):
"""Auto-answer 'yes' when prompted to confirm commands.
This sets the 'BREWBLOX_SKIP_CONFIRM' variable in .env.
You can still use the `brewblox-ctl [--dry-run|--verbose] COMMAND` arguments.
"""
utils.check_config()
utils.confirm_mode()
utils.setenv(const.SKIP_CONFIRM_KEY, value.lower())
def down():
"""Stop all services.
This wraps `docker-compose down --remove-orphans`
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh('{}docker-compose down --remove-orphans'.format(sudo))
def down(compose_args):
"""Stop all services.
This wraps `docker-compose down`
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh(f'{sudo}docker-compose down ' + ' '.join(list(compose_args)))
def up(compose_args):
"""Start all services.
This wraps `docker-compose up -d`
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh(f'{sudo}docker-compose up -d ' + ' '.join(list(compose_args)))
def restart():
"""Stop and start all services.
This wraps `docker-compose down --remove-orphans; docker-compose up -d`
Note: `docker-compose restart` also exists -
it restarts containers without recreating them.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh('{}docker-compose down --remove-orphans'.format(sudo))
sh('{}docker-compose up -d'.format(sudo))
def restart(compose_args):
"""Recreates all services.
This wraps `docker-compose up -d --force-recreate`
Note: `docker-compose restart` also exists -
it restarts containers without recreating them.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh(f'{sudo}docker-compose up -d --force-recreate ' +
' '.join(list(compose_args)))
def ports(http, https, mqtt):
"""Update used ports"""
utils.check_config()
utils.confirm_mode()
cfg = {
const.HTTP_PORT_KEY: http,
const.HTTPS_PORT_KEY: https,
const.MQTT_PORT_KEY: mqtt,
}
utils.info('Writing port settings to .env...')
for key, val in cfg.items():
utils.setenv(key, val)
def remove(ctx, services):
"""Remove a service."""
utils.check_config()
utils.confirm_mode()
config = utils.read_compose()
for name in services:
try:
del config['services'][name]
utils.info(f"Removed service '{name}'")
except KeyError:
utils.warn(f"Service '{name}' not found")
if services:
utils.write_compose(config)
restart_services(ctx, compose_args=['--remove-orphans'])
def from_couchdb():
"""Migrate configuration data from CouchDB to Redis.
In the 2020/09/22 release (config version 0.6.0)
Redis replaced CouchDB as configuration database.
This command copies the configuration data from CouchDB to Redis.
\b
Steps:
- Create CouchdDB container.
- Fetch data from CouchDB.
- Write data to Redis.
"""
utils.check_config()
utils.confirm_mode()
migration.migrate_couchdb()
def follow(services):
"""Show logs for one or more services.
This will start watching the logs for specified services.
Call without arguments to show logs for all running services.
Once started, press ctrl+C to stop.
Service name will be equal to those specified in docker-compose.log,
not the container name.
To follow logs for service 'spark-one':
\b
GOOD: `brewblox-ctl follow spark-one`
BAD: `brewblox-ctl follow brewblox_spark-one_1`
"""
utils.check_config()
sudo = utils.optsudo()
sh('{}docker-compose logs --follow {}'.format(sudo, ' '.join(services)))
def add_tilt(force):
"""
Create a service for the Tilt hydrometer.
The service listens for Bluetooth status updates from the Tilt,
and requires the host to have a Bluetooth receiver.
The empty ./tilt dir is created to hold calibration files.
"""
utils.check_config()
utils.confirm_mode()
name = 'tilt'
sudo = utils.optsudo()
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/brewblox-tilt:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'privileged': True,
'network_mode': 'host',
'volumes': [{
'type': 'bind',
'source': f'./{name}',
'target': '/share',
}],
'labels': [
'traefik.enable=false',
],
}
sh(f'mkdir -p ./{name}')
utils.write_compose(config)
click.echo(f'Added Tilt service `{name}`.')
click.echo('It will automatically show up in the UI.\n')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
def save(file, force):
"""Save Brewblox directory to snapshot.
This can be used to move Brewblox installations between hosts.
To load the snapshot, use `brewblox-ctl install --snapshot ARCHIVE`
or `brewblox-ctl snapshot load --file ARCHIVE`
Block data stored on Spark controllers is not included in the snapshot.
"""
utils.check_config()
utils.confirm_mode()
dir = Path('./').resolve()
if utils.path_exists(file):
if force or utils.confirm(f'`{file}` already exists. ' +
'Do you want to overwrite it?'):
sh(f'rm -f {file}')
else:
return
sh(f'sudo tar -C {dir.parent} --exclude .venv -czf {file} {dir.name}')
click.echo(Path(file).resolve())
def load(file):
"""Create Brewblox directory from snapshot.
This can be used to move Brewblox installations between hosts.
To create a snapshot, use `brewblox-ctl snapshot save`
"""
utils.check_config()
utils.confirm_mode()
dir = Path('./').resolve()
with TemporaryDirectory() as tmpdir:
utils.info(f'Extracting snapshot to {dir} directory...')
sh(f'tar -xzf {file} -C {tmpdir}')
content = list(Path(tmpdir).iterdir())
if utils.ctx_opts().dry_run:
content = ['brewblox']
if len(content) != 1:
raise ValueError(f'Multiple files found in snapshot: {content}')
sh('sudo rm -rf ./*')
# We need to explicitly include dotfiles in the mv glob
src = content[0]
sh(f'mv {src}/.[!.]* {src}/* {dir}/')
actions.install_ctl_package(download='missing')
def from_influxdb(target, duration, offset, services):
"""Migrate history data from InfluxDB to Victoria Metrics or file.
In config version 0.7.0 Victoria Metrics replaced InfluxDB as history database.
This command exports the history data from InfluxDB,
and then either immediately imports it to Victoria Metrics, or saves it to file.
By default, all services are migrated.
You can override this by listing the services you want to migrate.
When writing data to file, files are stored in the ./influxdb-export/ directory.
\b
Steps:
- Create InfluxDB container.
- Get list of services from InfluxDB. (Optional)
- Read data from InfluxDB.
- Write data to Victoria Metrics. (Optional)
- OR: write data to file. (Optional)
"""
utils.check_config()
utils.confirm_mode()
migration.migrate_influxdb(target, duration, list(services), list(offset))
def test_check_config(mocked_ext):
mocked_ext['getenv_'].side_effect = [
'1.2.3',
'',
'',
'',
'',
]
mocked_ext['input'].side_effect = [
'',
'no',
]
# version ok
assert utils.check_config()
# version nok, user ok
assert not utils.check_config(required=False)
# required version nok
with pytest.raises(SystemExit):
utils.check_config()
# version nok, user nok
with pytest.raises(SystemExit):
utils.check_config(required=False)
def test_check_config(mocker, mocked_ext):
m_is_brewblox_dir = mocker.patch(TESTED + '.is_brewblox_dir')
m_is_brewblox_dir.side_effect = [
'1.2.3',
'',
'',
'',
'',
]
mocked_ext['input'].side_effect = [
'',
'no',
]
# version ok
assert utils.check_config()
# version nok, user ok
assert not utils.check_config(required=False)
# required version nok
with pytest.raises(SystemExit):
utils.check_config()
# version nok, user nok
with pytest.raises(SystemExit):
utils.check_config(required=False)
def update(update_ctl, update_ctl_done, pull, update_system, migrate, prune,
from_version):
"""Download and apply updates.
This is the one-stop-shop for updating your Brewblox install.
You can use any of the options to fine-tune the update by enabling or disabling subroutines.
By default, all options are enabled.
--update-ctl/--no-update-ctl: Whether to download and install new versions of
of brewblox-ctl. If this flag is set, update will download the new version
and then restart itself. This way, the migrate is done with the latest version of brewblox-ctl.
If you're using dry run mode, you'll notice the hidden option --update-ctl-done.
You can use it to watch the rest of the update: it\'s a flag to avoid endless loops.
--pull/--no-pull. Whether to pull docker images.
This is useful if any of your services is using a local image (not from Docker Hub).
--update-system/--no-update-system determines whether
--migrate/--no-migrate. Updates regularly require changes to configuration.
Required changes are applied here.
--prune/--no-prune. Updates to docker images can leave unused old versions
on your system. These can be pruned to free up disk space.
This includes all images and volumes on your system, and not just those created by Brewblox.
\b
Steps:
- Check whether any system fixes must be applied.
- Update brewblox-ctl.
- Stop services.
- Update Avahi config.
- Update system packages.
- Migrate configuration files.
- Pull Docker images.
- Prune unused Docker images and volumes.
- Start services.
- Migrate service configuration.
- Write version number to .env file.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
prev_version = StrictVersion(from_version)
check_version(prev_version)
if update_ctl and not update_ctl_done:
utils.info('Updating brewblox-ctl...')
utils.pip_install('pip')
actions.install_ctl_package()
# Restart update - we just replaced the source code
sh(' '.join(
['python3 -m brewblox_ctl', *const.ARGS[1:], '--update-ctl-done']))
return
if update_ctl:
actions.uninstall_old_ctl_package()
actions.deploy_ctl_wrapper()
utils.info('Stopping services...')
sh(f'{sudo}docker-compose down')
if update_system:
actions.update_system_packages()
if migrate:
downed_migrate(prev_version)
if pull:
utils.info('Pulling docker images...')
sh(f'{sudo}docker-compose pull')
if prune:
utils.info('Pruning unused images...')
sh(f'{sudo}docker image prune -f > /dev/null')
utils.info('Pruning unused volumes...')
sh(f'{sudo}docker volume prune -f > /dev/null')
utils.info('Starting services...')
sh(f'{sudo}docker-compose up -d')
if migrate:
upped_migrate(prev_version)
utils.info(
f'Configuration version: {prev_version} -> {const.CURRENT_VERSION}'
)
utils.setenv(const.CFG_VERSION_KEY, const.CURRENT_VERSION)
def save(save_compose, ignore_spark_error):
"""Create a backup of Brewblox settings.
A zip archive containing JSON/YAML files is created in the ./backup/ directory.
The archive name will include current date and time to ensure uniqueness.
Restrictions:
- The backup is not exported to any kind of remote/cloud storage.
- The backup does not include history data.
- The backup does not include Docker images.
- The backup does not include custom configuration for third-party services.
To use this command in scripts, run it as `brewblox-ctl --quiet backup save`.
Its only output to stdout will be the absolute path to the created backup.
The command will fail if any of the Spark services could not be contacted.
As it does not make any destructive changes to configuration,
this command is not affected by --dry-run.
\b
Stored data:
- .env
- docker-compose.yml. (Optional)
- Datastore databases.
- Spark service blocks.
- Node-RED data.
- Mosquitto config files.
- Tilt config files.
\b
NOT stored:
- History data.
"""
utils.check_config()
urllib3.disable_warnings()
file = f'backup/brewblox_backup_{datetime.now().strftime("%Y%m%d_%H%M")}.zip'
with suppress(FileExistsError):
mkdir(Path('backup/').resolve())
store_url = utils.datastore_url()
utils.info('Waiting for the datastore...')
http.wait(store_url + '/ping', info_updates=True)
config = utils.read_compose()
sparks = [
k for k, v in config['services'].items()
if v.get('image', '').startswith('brewblox/brewblox-devcon-spark')
]
zipf = zipfile.ZipFile(file, 'w', zipfile.ZIP_DEFLATED)
# Always save .env
utils.info('Exporting .env')
zipf.write('.env')
# Always save datastore
utils.info('Exporting datastore')
resp = requests.post(store_url + '/mget',
json={
'namespace': '',
'filter': '*'
},
verify=False)
resp.raise_for_status()
zipf.writestr('global.redis.json', resp.text)
if save_compose:
utils.info('Exporting docker-compose.yml')
zipf.write('docker-compose.yml')
for spark in sparks:
utils.info(f'Exporting Spark blocks from `{spark}`')
resp = requests.post(f'{utils.host_url()}/{spark}/blocks/backup/save',
verify=False)
try:
resp.raise_for_status()
zipf.writestr(spark + '.spark.json', resp.text)
except Exception as ex:
if ignore_spark_error:
utils.info(f'Skipping Spark `{spark}` due to error: {str(ex)}')
else:
raise ex
for fname in [
*glob('node-red/*.js*'),
*glob('node-red/lib/**/*.js*'),
*glob('mosquitto/*.conf'),
*glob('tilt/*'),
]:
zipf.write(fname)
zipf.close()
click.echo(Path(file).resolve())
utils.info('Done!')
def load(archive, load_env, load_compose, load_datastore, load_spark,
load_node_red, load_mosquitto, load_tilt, update):
"""Load and apply Brewblox settings backup.
This function uses files generated by `brewblox-ctl backup save` as input.
You can use the --load-XXXX options to partially load the backup.
This does not attempt to merge data: it will overwrite current docker-compose.yml,
datastore entries, and Spark blocks.
Blocks on Spark services not in the backup file will not be affected.
If dry-run is enabled, it will echo all configuration from the backup archive.
Steps:
- Write .env
- Read .env values
- Write docker-compose.yml, run `docker-compose up`.
- Write all datastore files found in backup.
- Write all Spark blocks found in backup.
- Write Node-RED config files found in backup.
- Write Mosquitto config files found in backup.
- Run brewblox-ctl update
"""
utils.check_config()
utils.confirm_mode()
urllib3.disable_warnings()
sudo = utils.optsudo()
host_url = utils.host_url()
store_url = utils.datastore_url()
yaml = YAML()
zipf = zipfile.ZipFile(archive, 'r', zipfile.ZIP_DEFLATED)
available = zipf.namelist()
redis_file = 'global.redis.json'
couchdb_files = [v for v in available if v.endswith('.datastore.json')]
spark_files = [v for v in available if v.endswith('.spark.json')]
node_red_files = [v for v in available if v.startswith('node-red/')]
mosquitto_files = [v for v in available if v.startswith('mosquitto/')]
tilt_files = [v for v in available if v.startswith('tilt/')]
if load_env and '.env' in available:
utils.info('Loading .env file')
with NamedTemporaryFile('w') as tmp:
data = zipf.read('.env').decode()
utils.info('Writing .env')
utils.show_data('.env', data)
tmp.write(data)
tmp.flush()
sh(f'cp -f {tmp.name} .env')
utils.info('Reading .env values')
load_dotenv(Path('.env').resolve())
if load_compose:
if 'docker-compose.yml' in available:
utils.info('Loading docker-compose.yml')
config = yaml.load(zipf.read('docker-compose.yml').decode())
# Older services may still depend on the `datastore` service
# The `depends_on` config is useless anyway in a brewblox system
for svc in config['services'].values():
with suppress(KeyError):
del svc['depends_on']
utils.write_compose(config)
sh(f'{sudo}docker-compose up -d')
else:
utils.info('docker-compose.yml file not found in backup archive')
if load_datastore:
if redis_file in available or couchdb_files:
utils.info('Waiting for the datastore...')
sh(f'{const.CLI} http wait {store_url}/ping')
# Wipe UI/Automation, but leave Spark files
mdelete_cmd = '{} http post {}/mdelete --quiet -d \'{{"namespace":"{}", "filter":"*"}}\''
sh(mdelete_cmd.format(const.CLI, store_url, 'brewblox-ui-store'))
sh(mdelete_cmd.format(const.CLI, store_url, 'brewblox-automation'))
else:
utils.info('No datastore files found in backup archive')
if redis_file in available:
data = json.loads(zipf.read(redis_file).decode())
utils.info(
f'Loading {len(data["values"])} entries from Redis datastore')
mset(data)
# Backwards compatibility for UI/automation files from CouchDB
# The IDs here are formatted as {moduleId}__{objId}
# The module ID becomes part of the Redis namespace
for db in ['brewblox-ui-store', 'brewblox-automation']:
fname = f'{db}.datastore.json'
if fname not in available:
continue
docs = json.loads(zipf.read(fname).decode())
# Drop invalid names (not prefixed with module ID)
docs[:] = [d for d in docs if len(d['_id'].split('__', 1)) == 2]
# Add namespace / ID fields
for d in docs:
segments = d['_id'].split('__', 1)
d['namespace'] = f'{db}:{segments[0]}'
d['id'] = segments[1]
del d['_id']
utils.info(f'Loading {len(docs)} entries from database `{db}`')
mset({'values': docs})
# Backwards compatibility for Spark service files
# There is no module ID field here
spark_db = 'spark-service'
spark_fname = f'{spark_db}.datastore.json'
if spark_fname in available:
docs = json.loads(zipf.read(spark_fname).decode())
for d in docs:
d['namespace'] = spark_db
d['id'] = d['_id']
del d['_id']
utils.info(
f'Loading {len(docs)} entries from database `{spark_db}`')
mset({'values': docs})
if load_spark:
sudo = utils.optsudo()
if not spark_files:
utils.info('No Spark files found in backup archive')
for f in spark_files:
spark = f[:-len('.spark.json')]
utils.info(f'Writing blocks to Spark service `{spark}`')
with NamedTemporaryFile('w') as tmp:
data = json.loads(zipf.read(f).decode())
utils.show_data(spark, data)
json.dump(data, tmp)
tmp.flush()
sh(f'{const.CLI} http post {host_url}/{spark}/blocks/backup/load -f {tmp.name}'
)
sh(f'{sudo}docker-compose restart {spark}')
if load_node_red and node_red_files:
sudo = ''
if [getgid(), getuid()] != [1000, 1000]:
sudo = 'sudo '
with TemporaryDirectory() as tmpdir:
zipf.extractall(tmpdir, members=node_red_files)
sh('mkdir -p ./node-red')
sh(f'{sudo}chown 1000:1000 ./node-red/')
sh(f'{sudo}chown -R 1000:1000 {tmpdir}')
sh(f'{sudo}cp -rfp {tmpdir}/node-red/* ./node-red/')
if load_mosquitto and mosquitto_files:
zipf.extractall(members=mosquitto_files)
if load_tilt and tilt_files:
zipf.extractall(members=tilt_files)
zipf.close()
if update:
utils.info('Updating brewblox...')
sh(f'{const.CLI} update')
utils.info('Done!')
def log(add_compose, add_system, upload):
"""Generate and share log file for bug reports.
This command generates a comprehensive report on current system state and logs.
When reporting bugs, a termbin blink to the output is often the first thing asked for.
For best results, run when the services are still active.
Service logs are discarded after `brewblox-ctl down`.
Care is taken to prevent accidental leaks of confidential information.
Only known variables are read from .env,
and the `--no-add-compose` flag allows skipping compose configuration.
The latter is useful if the configuration contains passwords or tokens.
To review or edit the output, use the `--no-upload` flag.
The output will include instructions on how to manually upload the file.
\b
Steps:
- Create ./brewblox.log file.
- Append Brewblox .env variables.
- Append software version info.
- Append service logs.
- Append content of docker-compose.yml (optional).
- Append content of docker-compose.shared.yml (optional).
- Append blocks from Spark services.
- Append system diagnostics.
- Upload file to termbin.com for shareable link (optional).
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
# Create log
utils.info(f"Log file: {Path('./brewblox.log').resolve()}")
create()
append('date')
# Add .env values
utils.info('Writing Brewblox .env values...')
header('.env')
for key in ENV_KEYS:
append(f'echo "{key}={utils.getenv(key)}"')
# Add version info
utils.info('Writing software version info...')
header('Versions')
append('uname -a')
append('python3 --version')
append(f'{sudo}docker --version')
append(f'{sudo}docker-compose --version')
# Add active containers
utils.info('Writing active containers...')
header('Containers')
append(f'{sudo}docker-compose ps -a')
# Add service logs
try:
config_names = list(utils.read_compose()['services'].keys())
shared_names = list(utils.read_shared_compose()['services'].keys())
names = [n for n in config_names if n not in shared_names] + shared_names
for name in names:
utils.info(f'Writing {name} service logs...')
header(f'Service: {name}')
append(f'{sudo}docker-compose logs --timestamps --no-color --tail 200 {name}')
except Exception as ex:
append('echo ' + shlex.quote(type(ex).__name__ + ': ' + str(ex)))
# Add compose config
if add_compose:
utils.info('Writing docker-compose configuration...')
header('docker-compose.yml')
append('cat docker-compose.yml')
header('docker-compose.shared.yml')
append('cat docker-compose.shared.yml')
else:
utils.info('Skipping docker-compose configuration...')
# Add blocks
host_url = utils.host_url()
services = utils.list_services('brewblox/brewblox-devcon-spark')
for svc in services:
utils.info(f'Writing {svc} blocks...')
header(f'Blocks: {svc}')
append(f'{const.CLI} http post --pretty {host_url}/{svc}/blocks/all/read')
# Add system diagnostics
if add_system:
utils.info('Writing system diagnostics...')
header('docker info')
append(f'{sudo}docker info')
header('journalctl -u docker')
append('sudo journalctl -u docker | tail -100')
header('journalctl -u avahi-daemon')
append('sudo journalctl -u avahi-daemon | tail -100')
header('disk usage')
append('df -hl')
header('/var/log/syslog')
append('sudo tail -n 500 /var/log/syslog')
header('dmesg')
append('dmesg -T')
else:
utils.info('Skipping system diagnostics...')
# Upload
if upload:
click.echo(utils.file_netcat('termbin.com', 9999, Path('./brewblox.log')).decode())
else:
utils.info('Skipping upload. If you want to manually upload the log, run: ' +
click.style('brewblox-ctl termbin ./brewblox.log', fg='green'))
