def wifi():
"""Configure Spark Wifi settings.
This requires the Spark to be connected over USB.
\b
Steps:
- Stop running services.
- Look for valid USB device.
- Spark 2 / Spark 3:
- Trigger listening mode.
- Connect to device serial to set up Wifi.
- Spark 4:
- Print provisioning instructions.
"""
utils.confirm_mode()
while True:
particle_dev = usb.core.find(idVendor=const.VID_PARTICLE)
if particle_dev:
particle_wifi(particle_dev)
break
esp_dev = usb.core.find(idVendor=const.VID_ESPRESSIF,
idProduct=const.PID_ESP32)
if esp_dev:
esp_wifi()
break
utils.confirm_usb()
def add_plaato(name, token, force):
"""
Create a service for the Plaato airlock.
This will periodically query the Plaato server for current state.
An authentication token is required.
See https://plaato.io/apps/help-center#!hc-auth-token on how to get one.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/brewblox-plaato:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'environment': {
'PLAATO_AUTH': token,
},
'command': f'--name={name}',
}
utils.write_compose(config)
click.echo(f'Added Plaato service `{name}`.')
click.echo('This service publishes history data, but does not have a UI component.')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
def add_node_red(force):
"""
Create a service for Node-RED.
"""
utils.check_config()
utils.confirm_mode()
name = 'node-red'
sudo = utils.optsudo()
host = utils.host_ip()
port = utils.getenv(const.HTTPS_PORT_KEY)
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/node-red:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'volumes': [{
'type': 'bind',
'source': f'./{name}',
'target': '/data',
}]
}
sh(f'mkdir -p ./{name}')
if [getgid(), getuid()] != [1000, 1000]:
sh(f'sudo chown -R 1000:1000 ./{name}')
utils.write_compose(config)
click.echo(f'Added Node-RED service `{name}`.')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
click.echo(f'Visit https://{host}:{port}/{name} in your browser to load the editor.')
def avahi():
"""Edit avahi-daemon configuration on the host machine.
This includes disabling IPv6 mDNS support, and enabling mDNS reflection.
"""
utils.confirm_mode()
actions.edit_avahi_config()
def flash(release, pull):
"""Flash Spark firmware over USB.
This requires the Spark to be connected over USB.
After the first install, firmware updates can also be installed using the UI.
\b
Steps:
- Stop running services.
- Pull flasher image.
- Run flash command.
"""
utils.confirm_mode()
dev = find_usb_spark()
if dev.idProduct == const.PID_PHOTON:
utils.info('Flashing Spark 2...')
run_particle_flasher(release, pull, 'flash')
elif dev.idProduct == const.PID_P1:
utils.info('Flashing Spark 3...')
run_particle_flasher(release, pull, 'flash')
elif dev.idProduct == const.PID_ESP32:
utils.info('Flashing Spark 4...')
run_esp_flasher(release, pull)
else:
raise ValueError('Unknown USB device')
def kill(zombies):
"""Stop and remove all containers on this host.
This includes those not from Brewblox.
If the --zombies flag is set,
leftover processes that are still claiming a port will be forcibly removed.
Use this if you get "port is already allocated" errors after your system crashed.
"""
utils.confirm_mode()
sudo = utils.optsudo()
sh(f'{sudo}docker rm --force $({sudo}docker ps -aq)', check=False)
if zombies:
# We can't use psutil for this, as we need root rights to get pids
if not utils.command_exists('netstat'):
utils.warn(
'Command `netstat` not found. Please install it by running:')
utils.warn('')
utils.warn(
' sudo apt-get update && sudo apt-get install net-tools')
utils.warn('')
return
procs = re.findall(r'(\d+)/docker-proxy',
sh('sudo netstat -pna', capture=True))
if procs:
utils.info(f'Removing {len(procs)} zombies...')
sh('sudo service docker stop')
sh([f'sudo kill -9 {proc}' for proc in procs])
sh('sudo service docker start')
def set_env(key, value):
"""Set a .env variable.
The value will be added to the .env file. You can set new variables with this.
"""
utils.check_config()
utils.confirm_mode()
utils.setenv(key, value)
def skip_confirm(value):
"""Auto-answer 'yes' when prompted to confirm commands.
This sets the 'BREWBLOX_SKIP_CONFIRM' variable in .env.
You can still use the `brewblox-ctl [--dry-run|--verbose] COMMAND` arguments.
"""
utils.check_config()
utils.confirm_mode()
utils.setenv(const.SKIP_CONFIRM_KEY, value.lower())
def down():
"""Stop all services.
This wraps `docker-compose down --remove-orphans`
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh('{}docker-compose down --remove-orphans'.format(sudo))
def kill():
"""Stop and remove all containers on this computer.
This includes those not from Brewblox.
"""
utils.confirm_mode()
sudo = utils.optsudo()
sh('{}docker rm --force $({}docker ps -aq)'.format(sudo, sudo),
check=False)
def down(compose_args):
"""Stop all services.
This wraps `docker-compose down`
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh(f'{sudo}docker-compose down ' + ' '.join(list(compose_args)))
def up(compose_args):
"""Start all services.
This wraps `docker-compose up -d`
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh(f'{sudo}docker-compose up -d ' + ' '.join(list(compose_args)))
def makecert(dir, release):
"""Generate a self-signed SSL certificate.
\b
Steps:
- Create directory if it does not exist.
- Create brewblox.crt and brewblox.key files.
"""
utils.confirm_mode()
actions.makecert(dir, release)
def ipv6(config_file):
"""Fix IPv6 support on the host machine.
Reason: https://github.com/docker/for-linux/issues/914
Unlike globally disabling IPv6 support on the host,
this has no impact outside the Docker environment.
Some hosts (Synology) may be using a custom location for the daemon config file.
If the --config-file argument is not set, the --config-file argument for the active docker daemon is used.
If that is not set, the default /etc/docker/daemon.json is used.
"""
utils.confirm_mode()
actions.fix_ipv6(config_file)
def restart():
"""Stop and start all services.
This wraps `docker-compose down --remove-orphans; docker-compose up -d`
Note: `docker-compose restart` also exists -
it restarts containers without recreating them.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh('{}docker-compose down --remove-orphans'.format(sudo))
sh('{}docker-compose up -d'.format(sudo))
def restart(compose_args):
"""Recreates all services.
This wraps `docker-compose up -d --force-recreate`
Note: `docker-compose restart` also exists -
it restarts containers without recreating them.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
sh(f'{sudo}docker-compose up -d --force-recreate ' +
' '.join(list(compose_args)))
def ports(http, https, mqtt):
"""Update used ports"""
utils.check_config()
utils.confirm_mode()
cfg = {
const.HTTP_PORT_KEY: http,
const.HTTPS_PORT_KEY: https,
const.MQTT_PORT_KEY: mqtt,
}
utils.info('Writing port settings to .env...')
for key, val in cfg.items():
utils.setenv(key, val)
def particle(release, pull, command):
"""Start a Docker container with access to the Particle CLI.
This requires the Spark to be connected over USB.
\b
Steps:
- Stop running services.
- Pull flasher image.
- Start flasher image.
"""
utils.confirm_mode()
utils.info('Starting Particle image...')
utils.info("Type 'exit' and press enter to exit the shell")
run_particle_flasher(release, pull, command)
def remove(ctx, services):
"""Remove a service."""
utils.check_config()
utils.confirm_mode()
config = utils.read_compose()
for name in services:
try:
del config['services'][name]
utils.info(f"Removed service '{name}'")
except KeyError:
utils.warn(f"Service '{name}' not found")
if services:
utils.write_compose(config)
restart_services(ctx, compose_args=['--remove-orphans'])
def from_couchdb():
"""Migrate configuration data from CouchDB to Redis.
In the 2020/09/22 release (config version 0.6.0)
Redis replaced CouchDB as configuration database.
This command copies the configuration data from CouchDB to Redis.
\b
Steps:
- Create CouchdDB container.
- Fetch data from CouchDB.
- Write data to Redis.
"""
utils.check_config()
utils.confirm_mode()
migration.migrate_couchdb()
def disable_ipv6():
"""Disable IPv6 support on the host machine.
Reason: https://github.com/docker/for-linux/issues/914
Should only be used if your services are having stability issues
"""
utils.confirm_mode()
is_disabled = sh('cat /proc/sys/net/ipv6/conf/all/disable_ipv6', capture=True).strip()
if is_disabled == '1':
utils.info('IPv6 is already disabled')
elif is_disabled == '0' or utils.ctx_opts().dry_run:
utils.info('Disabling IPv6...')
sh('echo "net.ipv6.conf.all.disable_ipv6 = 1" | sudo tee -a /etc/sysctl.conf')
sh('echo "net.ipv6.conf.default.disable_ipv6 = 1" | sudo tee -a /etc/sysctl.conf')
sh('echo "net.ipv6.conf.lo.disable_ipv6 = 1" | sudo tee -a /etc/sysctl.conf')
sh('sudo sysctl -p')
else:
utils.info('Invalid result when checking IPv6 status: ' + is_disabled)
def flash(release, pull):
"""Flash firmware on Spark.
This requires the Spark to be connected over USB.
After the first install, firmware updates can also be installed using the UI.
\b
Steps:
- Stop running services.
- Pull flasher image.
- Run flash command.
"""
utils.confirm_mode()
utils.confirm_usb()
prepare_flasher(release, pull)
utils.info('Flashing Spark...')
run_flasher(release, 'flash')
def add_tilt(force):
"""
Create a service for the Tilt hydrometer.
The service listens for Bluetooth status updates from the Tilt,
and requires the host to have a Bluetooth receiver.
The empty ./tilt dir is created to hold calibration files.
"""
utils.check_config()
utils.confirm_mode()
name = 'tilt'
sudo = utils.optsudo()
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/brewblox-tilt:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'privileged': True,
'network_mode': 'host',
'volumes': [{
'type': 'bind',
'source': f'./{name}',
'target': '/share',
}],
'labels': [
'traefik.enable=false',
],
}
sh(f'mkdir -p ./{name}')
utils.write_compose(config)
click.echo(f'Added Tilt service `{name}`.')
click.echo('It will automatically show up in the UI.\n')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
def save(file, force):
"""Save Brewblox directory to snapshot.
This can be used to move Brewblox installations between hosts.
To load the snapshot, use `brewblox-ctl install --snapshot ARCHIVE`
or `brewblox-ctl snapshot load --file ARCHIVE`
Block data stored on Spark controllers is not included in the snapshot.
"""
utils.check_config()
utils.confirm_mode()
dir = Path('./').resolve()
if utils.path_exists(file):
if force or utils.confirm(f'`{file}` already exists. ' +
'Do you want to overwrite it?'):
sh(f'rm -f {file}')
else:
return
sh(f'sudo tar -C {dir.parent} --exclude .venv -czf {file} {dir.name}')
click.echo(Path(file).resolve())
def load(file):
"""Create Brewblox directory from snapshot.
This can be used to move Brewblox installations between hosts.
To create a snapshot, use `brewblox-ctl snapshot save`
"""
utils.check_config()
utils.confirm_mode()
dir = Path('./').resolve()
with TemporaryDirectory() as tmpdir:
utils.info(f'Extracting snapshot to {dir} directory...')
sh(f'tar -xzf {file} -C {tmpdir}')
content = list(Path(tmpdir).iterdir())
if utils.ctx_opts().dry_run:
content = ['brewblox']
if len(content) != 1:
raise ValueError(f'Multiple files found in snapshot: {content}')
sh('sudo rm -rf ./*')
# We need to explicitly include dotfiles in the mv glob
src = content[0]
sh(f'mv {src}/.[!.]* {src}/* {dir}/')
actions.install_ctl_package(download='missing')
def from_influxdb(target, duration, offset, services):
"""Migrate history data from InfluxDB to Victoria Metrics or file.
In config version 0.7.0 Victoria Metrics replaced InfluxDB as history database.
This command exports the history data from InfluxDB,
and then either immediately imports it to Victoria Metrics, or saves it to file.
By default, all services are migrated.
You can override this by listing the services you want to migrate.
When writing data to file, files are stored in the ./influxdb-export/ directory.
\b
Steps:
- Create InfluxDB container.
- Get list of services from InfluxDB. (Optional)
- Read data from InfluxDB.
- Write data to Victoria Metrics. (Optional)
- OR: write data to file. (Optional)
"""
utils.check_config()
utils.confirm_mode()
migration.migrate_influxdb(target, duration, list(services), list(offset))
def add_spark(name,
discover_now,
device_id,
discovery_type,
device_host,
command,
force,
release,
simulation):
"""
Create or update a Spark service.
If you run brewblox-ctl add-spark without any arguments,
it will prompt you for required info, and then create a sensibly configured service.
If you want to fine-tune your service configuration, multiple arguments are available.
For a detailed explanation: https://brewblox.netlify.com/user/connect_settings.html
"""
# utils.check_config()
utils.confirm_mode()
image_name = 'brewblox/brewblox-devcon-spark'
sudo = utils.optsudo()
config = utils.read_compose()
if not force:
check_duplicate(config, name)
for (nm, svc) in config['services'].items():
img = svc.get('image', '')
cmd = svc.get('command', '')
if not any([
nm == name,
not img.startswith(image_name),
'--device-id' in cmd,
'--device-host' in cmd,
'--simulation' in cmd,
]):
utils.warn(f'The existing Spark service `{nm}` does not have any connection settings.')
utils.warn('It will connect to any controller it can find.')
utils.warn('This may cause multiple services to connect to the same controller.')
utils.warn(f'To reconfigure `{nm}`, please run:')
utils.warn('')
utils.warn(f' brewblox-ctl add-spark -f --name {nm}')
utils.warn('')
utils.select('Press ENTER to continue or Ctrl-C to exit')
if discover_now and not simulation and not device_id:
if device_host:
dev = find_device_by_host(device_host)
else:
dev = choose_device(discovery_type, config)
if dev:
device_id = dev['id']
else:
# We have no device ID, and no device host. Avoid a wildcard service
click.echo('No valid combination of device ID and device host.')
raise SystemExit(1)
commands = [
'--name=' + name,
'--discovery=' + discovery_type,
]
if device_id:
commands += ['--device-id=' + device_id]
if device_host:
commands += ['--device-host=' + device_host]
if simulation:
commands += ['--simulation']
if command:
commands += [command]
config['services'][name] = {
'image': f'{image_name}:{utils.docker_tag(release)}',
'privileged': True,
'restart': 'unless-stopped',
'command': ' '.join(commands)
}
if simulation:
mount_dir = f'simulator__{name}'
config['services'][name]['volumes'] = [{
'type': 'bind',
'source': f'./{mount_dir}',
'target': '/app/simulator'
}]
sh(f'mkdir -m 777 -p {mount_dir}')
utils.write_compose(config)
click.echo(f'Added Spark service `{name}`.')
click.echo('It will automatically show up in the UI.\n')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
def install(ctx: click.Context, snapshot_file):
"""Install Brewblox and its dependencies.
Brewblox can be installed multiple times on the same computer.
Settings and databases are stored in a Brewblox directory.
This command also installs system-wide dependencies.
A reboot is required after installing docker, or adding the user to the 'docker' group.
By default, `brewblox-ctl install` attempts to download packages using the apt package manager.
If you are using a system without apt (eg. Synology NAS), this step will be skipped.
You will need to manually install any missing libraries.
When using the `--snapshot ARCHIVE` option, no dir is created.
Instead, the directory in the snapshot is extracted.
It will be renamed to the desired name of the Brewblox directory.
\b
Steps:
- Ask confirmation for installation steps.
- Install apt packages.
- Install docker.
- Add user to 'docker' group.
- Fix host IPv6 settings.
- Disable host-wide mDNS reflection.
- Set variables in .env file.
- If snapshot provided:
- Load configuration from snapshot.
- Else:
- Check for port conflicts.
- Create docker-compose configuration files.
- Create datastore (Redis) directory.
- Create history (Victoria) directory.
- Create gateway (Traefik) directory.
- Create SSL certificates.
- Create eventbus (Mosquitto) directory.
- Set version number in .env file.
- Pull docker images.
- Reboot if needed.
"""
utils.confirm_mode()
user = utils.getenv('USER')
opts = InstallOptions()
opts.check_confirm_opts()
opts.check_system_opts()
opts.check_docker_opts()
opts.check_reboot_opts()
if not snapshot_file:
opts.check_init_opts()
# Install Apt packages
if opts.apt_install:
utils.info('Installing apt packages...')
apt_deps = ' '.join(const.APT_DEPENDENCIES)
sh([
'sudo apt-get update',
'sudo apt-get upgrade -y',
f'sudo apt-get install -y {apt_deps}',
])
else:
utils.info('Skipped: apt-get install.')
# Install docker
if opts.docker_install:
utils.info('Installing docker...')
sh('curl -sL get.docker.com | sh', check=False)
else:
utils.info('Skipped: docker install.')
# Add user to 'docker' group
if opts.docker_group_add:
utils.info(f"Adding {user} to 'docker' group...")
sh('sudo usermod -aG docker $USER')
else:
utils.info(f"Skipped: adding {user} to 'docker' group.")
# Always apply actions
actions.disable_ssh_accept_env()
actions.fix_ipv6(None, False)
actions.edit_avahi_config()
actions.add_particle_udev_rules()
actions.uninstall_old_ctl_package()
actions.deploy_ctl_wrapper()
# Set variables in .env file
# Set version number to 0.0.0 until snapshot load / init is done
utils.info('Setting .env values...')
utils.setenv(const.CFG_VERSION_KEY, '0.0.0')
utils.setenv(const.SKIP_CONFIRM_KEY, str(opts.skip_confirm))
for key, default_val in const.ENV_DEFAULTS.items():
utils.setenv(key, utils.getenv(key, default_val))
# Install process splits here
# Either load all config files from snapshot or run init
sudo = utils.optsudo()
if snapshot_file:
ctx.invoke(snapshot.load, file=snapshot_file)
else:
release = utils.getenv('BREWBLOX_RELEASE')
utils.info('Checking for port conflicts...')
actions.check_ports()
utils.info('Copying docker-compose.shared.yml...')
sh(f'cp -f {const.CONFIG_DIR}/docker-compose.shared.yml ./')
if opts.init_compose:
utils.info('Copying docker-compose.yml...')
sh(f'cp -f {const.CONFIG_DIR}/docker-compose.yml ./')
# Stop after we're sure we have a compose file
utils.info('Stopping services...')
sh(f'{sudo}docker-compose down')
if opts.init_datastore:
utils.info('Creating datastore directory...')
sh('sudo rm -rf ./redis/; mkdir ./redis/')
if opts.init_history:
utils.info('Creating history directory...')
sh('sudo rm -rf ./victoria/; mkdir ./victoria/')
if opts.init_gateway:
utils.info('Creating gateway directory...')
sh('sudo rm -rf ./traefik/; mkdir ./traefik/')
utils.info('Creating SSL certificate...')
actions.makecert('./traefik', release)
if opts.init_eventbus:
utils.info('Creating mosquitto config directory...')
sh('sudo rm -rf ./mosquitto/; mkdir ./mosquitto/')
# Always copy cert config to traefik dir
sh(f'cp -f {const.CONFIG_DIR}/traefik-cert.yaml ./traefik/')
# Init done - now set CFG version
utils.setenv(const.CFG_VERSION_KEY, const.CURRENT_VERSION)
if opts.docker_pull:
utils.info('Pulling docker images...')
sh(f'{sudo}docker-compose pull')
utils.info('All done!')
# Reboot
if opts.reboot_needed:
if opts.prompt_reboot:
utils.info('Press ENTER to reboot.')
input()
else:
utils.info('Rebooting in 10 seconds...')
sleep(10)
sh('sudo reboot')
def log(add_compose, add_system, upload):
"""Generate and share log file for bug reports.
This command generates a comprehensive report on current system state and logs.
When reporting bugs, a termbin blink to the output is often the first thing asked for.
For best results, run when the services are still active.
Service logs are discarded after `brewblox-ctl down`.
Care is taken to prevent accidental leaks of confidential information.
Only known variables are read from .env,
and the `--no-add-compose` flag allows skipping compose configuration.
The latter is useful if the configuration contains passwords or tokens.
To review or edit the output, use the `--no-upload` flag.
The output will include instructions on how to manually upload the file.
\b
Steps:
- Create ./brewblox.log file.
- Append Brewblox .env variables.
- Append software version info.
- Append service logs.
- Append content of docker-compose.yml (optional).
- Append content of docker-compose.shared.yml (optional).
- Append blocks from Spark services.
- Append system diagnostics.
- Upload file to termbin.com for shareable link (optional).
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
# Create log
utils.info(f"Log file: {Path('./brewblox.log').resolve()}")
create()
append('date')
# Add .env values
utils.info('Writing Brewblox .env values...')
header('.env')
for key in ENV_KEYS:
append(f'echo "{key}={utils.getenv(key)}"')
# Add version info
utils.info('Writing software version info...')
header('Versions')
append('uname -a')
append('python3 --version')
append(f'{sudo}docker --version')
append(f'{sudo}docker-compose --version')
# Add active containers
utils.info('Writing active containers...')
header('Containers')
append(f'{sudo}docker-compose ps -a')
# Add service logs
try:
config_names = list(utils.read_compose()['services'].keys())
shared_names = list(utils.read_shared_compose()['services'].keys())
names = [n for n in config_names if n not in shared_names] + shared_names
for name in names:
utils.info(f'Writing {name} service logs...')
header(f'Service: {name}')
append(f'{sudo}docker-compose logs --timestamps --no-color --tail 200 {name}')
except Exception as ex:
append('echo ' + shlex.quote(type(ex).__name__ + ': ' + str(ex)))
# Add compose config
if add_compose:
utils.info('Writing docker-compose configuration...')
header('docker-compose.yml')
append('cat docker-compose.yml')
header('docker-compose.shared.yml')
append('cat docker-compose.shared.yml')
else:
utils.info('Skipping docker-compose configuration...')
# Add blocks
host_url = utils.host_url()
services = utils.list_services('brewblox/brewblox-devcon-spark')
for svc in services:
utils.info(f'Writing {svc} blocks...')
header(f'Blocks: {svc}')
append(f'{const.CLI} http post --pretty {host_url}/{svc}/blocks/all/read')
# Add system diagnostics
if add_system:
utils.info('Writing system diagnostics...')
header('docker info')
append(f'{sudo}docker info')
header('journalctl -u docker')
append('sudo journalctl -u docker | tail -100')
header('journalctl -u avahi-daemon')
append('sudo journalctl -u avahi-daemon | tail -100')
header('disk usage')
append('df -hl')
header('/var/log/syslog')
append('sudo tail -n 500 /var/log/syslog')
header('dmesg')
append('dmesg -T')
else:
utils.info('Skipping system diagnostics...')
# Upload
if upload:
click.echo(utils.file_netcat('termbin.com', 9999, Path('./brewblox.log')).decode())
else:
utils.info('Skipping upload. If you want to manually upload the log, run: ' +
click.style('brewblox-ctl termbin ./brewblox.log', fg='green'))
def load(archive, load_env, load_compose, load_datastore, load_spark,
load_node_red, load_mosquitto, load_tilt, update):
"""Load and apply Brewblox settings backup.
This function uses files generated by `brewblox-ctl backup save` as input.
You can use the --load-XXXX options to partially load the backup.
This does not attempt to merge data: it will overwrite current docker-compose.yml,
datastore entries, and Spark blocks.
Blocks on Spark services not in the backup file will not be affected.
If dry-run is enabled, it will echo all configuration from the backup archive.
Steps:
- Write .env
- Read .env values
- Write docker-compose.yml, run `docker-compose up`.
- Write all datastore files found in backup.
- Write all Spark blocks found in backup.
- Write Node-RED config files found in backup.
- Write Mosquitto config files found in backup.
- Run brewblox-ctl update
"""
utils.check_config()
utils.confirm_mode()
urllib3.disable_warnings()
sudo = utils.optsudo()
host_url = utils.host_url()
store_url = utils.datastore_url()
yaml = YAML()
zipf = zipfile.ZipFile(archive, 'r', zipfile.ZIP_DEFLATED)
available = zipf.namelist()
redis_file = 'global.redis.json'
couchdb_files = [v for v in available if v.endswith('.datastore.json')]
spark_files = [v for v in available if v.endswith('.spark.json')]
node_red_files = [v for v in available if v.startswith('node-red/')]
mosquitto_files = [v for v in available if v.startswith('mosquitto/')]
tilt_files = [v for v in available if v.startswith('tilt/')]
if load_env and '.env' in available:
utils.info('Loading .env file')
with NamedTemporaryFile('w') as tmp:
data = zipf.read('.env').decode()
utils.info('Writing .env')
utils.show_data('.env', data)
tmp.write(data)
tmp.flush()
sh(f'cp -f {tmp.name} .env')
utils.info('Reading .env values')
load_dotenv(Path('.env').resolve())
if load_compose:
if 'docker-compose.yml' in available:
utils.info('Loading docker-compose.yml')
config = yaml.load(zipf.read('docker-compose.yml').decode())
# Older services may still depend on the `datastore` service
# The `depends_on` config is useless anyway in a brewblox system
for svc in config['services'].values():
with suppress(KeyError):
del svc['depends_on']
utils.write_compose(config)
sh(f'{sudo}docker-compose up -d')
else:
utils.info('docker-compose.yml file not found in backup archive')
if load_datastore:
if redis_file in available or couchdb_files:
utils.info('Waiting for the datastore...')
sh(f'{const.CLI} http wait {store_url}/ping')
# Wipe UI/Automation, but leave Spark files
mdelete_cmd = '{} http post {}/mdelete --quiet -d \'{{"namespace":"{}", "filter":"*"}}\''
sh(mdelete_cmd.format(const.CLI, store_url, 'brewblox-ui-store'))
sh(mdelete_cmd.format(const.CLI, store_url, 'brewblox-automation'))
else:
utils.info('No datastore files found in backup archive')
if redis_file in available:
data = json.loads(zipf.read(redis_file).decode())
utils.info(
f'Loading {len(data["values"])} entries from Redis datastore')
mset(data)
# Backwards compatibility for UI/automation files from CouchDB
# The IDs here are formatted as {moduleId}__{objId}
# The module ID becomes part of the Redis namespace
for db in ['brewblox-ui-store', 'brewblox-automation']:
fname = f'{db}.datastore.json'
if fname not in available:
continue
docs = json.loads(zipf.read(fname).decode())
# Drop invalid names (not prefixed with module ID)
docs[:] = [d for d in docs if len(d['_id'].split('__', 1)) == 2]
# Add namespace / ID fields
for d in docs:
segments = d['_id'].split('__', 1)
d['namespace'] = f'{db}:{segments[0]}'
d['id'] = segments[1]
del d['_id']
utils.info(f'Loading {len(docs)} entries from database `{db}`')
mset({'values': docs})
# Backwards compatibility for Spark service files
# There is no module ID field here
spark_db = 'spark-service'
spark_fname = f'{spark_db}.datastore.json'
if spark_fname in available:
docs = json.loads(zipf.read(spark_fname).decode())
for d in docs:
d['namespace'] = spark_db
d['id'] = d['_id']
del d['_id']
utils.info(
f'Loading {len(docs)} entries from database `{spark_db}`')
mset({'values': docs})
if load_spark:
sudo = utils.optsudo()
if not spark_files:
utils.info('No Spark files found in backup archive')
for f in spark_files:
spark = f[:-len('.spark.json')]
utils.info(f'Writing blocks to Spark service `{spark}`')
with NamedTemporaryFile('w') as tmp:
data = json.loads(zipf.read(f).decode())
utils.show_data(spark, data)
json.dump(data, tmp)
tmp.flush()
sh(f'{const.CLI} http post {host_url}/{spark}/blocks/backup/load -f {tmp.name}'
)
sh(f'{sudo}docker-compose restart {spark}')
if load_node_red and node_red_files:
sudo = ''
if [getgid(), getuid()] != [1000, 1000]:
sudo = 'sudo '
with TemporaryDirectory() as tmpdir:
zipf.extractall(tmpdir, members=node_red_files)
sh('mkdir -p ./node-red')
sh(f'{sudo}chown 1000:1000 ./node-red/')
sh(f'{sudo}chown -R 1000:1000 {tmpdir}')
sh(f'{sudo}cp -rfp {tmpdir}/node-red/* ./node-red/')
if load_mosquitto and mosquitto_files:
zipf.extractall(members=mosquitto_files)
if load_tilt and tilt_files:
zipf.extractall(members=tilt_files)
zipf.close()
if update:
utils.info('Updating brewblox...')
sh(f'{const.CLI} update')
utils.info('Done!')
