def check_ports():
if utils.path_exists('./docker-compose.yml'):
utils.info('Stopping services...')
sh('{}docker-compose down --remove-orphans'.format(utils.optsudo()))
ports = [
utils.getenv(key, const.ENV_DEFAULTS[key]) for key in [
const.HTTP_PORT_KEY,
const.HTTPS_PORT_KEY,
const.MDNS_PORT_KEY,
]
]
utils.info('Checking ports...')
retv = sh('sudo netstat -tulpn', capture=True)
lines = retv.split('\n')
used_ports = []
used_lines = []
for port in ports:
for line in lines:
if re.match(r'.*(:::|0.0.0.0:){}\s.*'.format(port), line):
used_ports.append(port)
used_lines.append(line)
break
if used_ports:
utils.warn(
'Port(s) {} already in use. '.format(', '.join(used_ports)) +
"Run 'brewblox-ctl service ports' to configure Brewblox ports.")
for line in used_lines:
utils.warn(line)
if not utils.confirm('Do you want to continue?'):
raise SystemExit(1)
def mset(data):
with NamedTemporaryFile('w') as tmp:
utils.show_data('datastore', data)
json.dump(data, tmp)
tmp.flush()
sh(f'{const.CLI} http post --quiet {utils.datastore_url()}/mset -f {tmp.name}'
)
def add_plaato(name, token, force):
"""
Create a service for the Plaato airlock.
This will periodically query the Plaato server for current state.
An authentication token is required.
See https://plaato.io/apps/help-center#!hc-auth-token on how to get one.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/brewblox-plaato:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'environment': {
'PLAATO_AUTH': token,
},
'command': f'--name={name}',
}
utils.write_compose(config)
click.echo(f'Added Plaato service `{name}`.')
click.echo('This service publishes history data, but does not have a UI component.')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
def migrate_ipv6_fix():
# Undo disable-ipv6
sh('sudo sed -i "/net.ipv6.*.disable_ipv6 = 1/d" /etc/sysctl.conf',
check=False)
# Enable ipv6 in docker daemon config
utils.enable_ipv6()
def editor(ctx, port):
"""Run web-based docker-compose.yml editor.
This will start a new docker container listening on a host port (default: 8300).
Navigate there in your browser to access the GUI for editing docker-compose.yml.
When you're done editing, save your file in the GUI, and press Ctrl+C in the terminal.
"""
utils.check_config()
utils.confirm_mode()
orig = utils.read_file('docker-compose.yml')
sudo = utils.optsudo()
host_ip = utils.host_ip()
editor = 'brewblox/brewblox-web-editor:{}'.format(utils.docker_tag())
utils.info('Pulling image...')
sh('{}docker pull {}'.format(sudo, editor))
try:
utils.info('Starting editor...')
sh('{}docker run'.format(sudo) + ' --rm --init' +
' -p "{}:8300"'.format(port) + ' -v "$(pwd):/app/config"' +
' {}'.format(editor) + ' --host-address {}'.format(host_ip) +
' --host-port {}'.format(port))
except KeyboardInterrupt: # pragma: no cover
pass
if orig != utils.read_file('docker-compose.yml'):
utils.info('Configuration changes detected.')
restart_services(ctx)
def check_ports():
if utils.path_exists('./docker-compose.yml'):
utils.info('Stopping services...')
sh(f'{utils.optsudo()}docker-compose down')
ports = [
int(utils.getenv(key, const.ENV_DEFAULTS[key])) for key in [
const.HTTP_PORT_KEY,
const.HTTPS_PORT_KEY,
const.MQTT_PORT_KEY,
]
]
try:
port_connnections = [
conn for conn in psutil.net_connections()
if conn.laddr.ip in ['::', '0.0.0.0'] and conn.laddr.port in ports
]
except psutil.AccessDenied:
utils.warn(
'Unable to read network connections. You need to run `netstat` or `lsof` manually.'
)
port_connnections = []
if port_connnections:
port_str = ', '.join(
set(str(conn.laddr.port) for conn in port_connnections))
utils.warn(f'Port(s) {port_str} already in use.')
utils.warn(
'Run `brewblox-ctl service ports` to configure Brewblox ports.')
if not utils.confirm('Do you want to continue?'):
raise SystemExit(1)
def particle_wifi(dev: usb.core.Device):
if utils.ctx_opts().dry_run:
utils.info('Dry run: skipping activation of Spark listening mode')
else:
dev.reset()
# Magic numbers for the USB control call
HOST_TO_DEVICE = 0x40 # bmRequestType
REQUEST_INIT = 1 # bRequest
REQUEST_SEND = 3 # bRequest
PARTICLE_LISTEN_INDEX = 70 # wIndex
PARTICLE_LISTEN_VALUE = 0 # wValue
PARTICLE_BUF_SIZE = 64 # wLength
dev.ctrl_transfer(HOST_TO_DEVICE, REQUEST_INIT, PARTICLE_LISTEN_VALUE,
PARTICLE_LISTEN_INDEX, PARTICLE_BUF_SIZE)
dev.ctrl_transfer(HOST_TO_DEVICE, REQUEST_SEND, PARTICLE_LISTEN_VALUE,
PARTICLE_LISTEN_INDEX, PARTICLE_BUF_SIZE)
sleep(LISTEN_MODE_WAIT_S)
serial = usb.util.get_string(dev, dev.iSerialNumber)
path = next(
Path('/dev/serial/by-id').glob(f'*{serial}*'), Path('/dev/ttyACM0'))
utils.info('Press w to start Wifi configuration.')
utils.info('Press Ctrl + ] to cancel.')
utils.info('The Spark must be restarted after canceling.')
sh(f'pyserial-miniterm -q {path.resolve()} 2>/dev/null')
def apply_shared():
"""Apply docker-compose.shared.yml from data directory"""
sh('cp -f {}/docker-compose.shared.yml ./'.format(const.CONFIG_DIR))
shared_cfg = utils.read_shared_compose()
usr_cfg = utils.read_compose()
usr_cfg['version'] = shared_cfg['version']
utils.write_compose(usr_cfg)
def install_ctl_package(download: str = 'always'): # always | missing | never
exists = utils.path_exists('./brewblox-ctl.tar.gz')
release = utils.getenv(const.CTL_RELEASE_KEY) or utils.getenv(
const.RELEASE_KEY)
if download == 'always' or download == 'missing' and not exists:
sh(f'wget -q -O ./brewblox-ctl.tar.gz https://brewblox.blob.core.windows.net/ctl/{release}/brewblox-ctl.tar.gz'
)
sh('python3 -m pip install ./brewblox-ctl.tar.gz')
def mset(data):
with NamedTemporaryFile('w') as tmp:
utils.show_data(data)
json.dump(data, tmp)
tmp.flush()
sh('{} http post --quiet {}/mset -f {}'.format(const.CLI,
utils.datastore_url(),
tmp.name))
def apply_config():
"""Apply system-defined configuration from config dir"""
sh('cp -f {}/traefik-cert.yaml ./traefik/'.format(const.CONFIG_DIR))
sh('cp -f {}/docker-compose.shared.yml ./'.format(const.CONFIG_DIR))
shared_cfg = utils.read_shared_compose()
usr_cfg = utils.read_compose()
usr_cfg['version'] = shared_cfg['version']
utils.write_compose(usr_cfg)
def apply_config_files():
"""Apply system-defined configuration from config dir"""
utils.info('Updating configuration files...')
sh(f'cp -f {const.CONFIG_DIR}/traefik-cert.yaml ./traefik/')
sh(f'cp -f {const.CONFIG_DIR}/docker-compose.shared.yml ./')
shared_cfg = utils.read_shared_compose()
usr_cfg = utils.read_compose()
usr_cfg['version'] = shared_cfg['version']
utils.write_compose(usr_cfg)
def upped_migrate(prev_version):
"""Migration commands to be executed after the services have been started"""
# Always run history configure
history_url = utils.history_url()
sh('{} http wait {}/ping'.format(const.CLI, history_url))
sh('{} http post --quiet {}/configure'.format(const.CLI, history_url))
if prev_version < StrictVersion('0.6.0'):
utils.info('Migrating datastore from CouchDB to Redis...')
datastore_migrate_redis()
def downed_migrate(prev_version):
"""Migration commands to be executed without any running services"""
if prev_version < StrictVersion('0.2.0'):
# Breaking changes: Influx downsampling model overhaul
# Old data is completely incompatible
utils.select(
'Upgrading to version >=0.2.0 requires a complete reset of your history data. '
+ "We'll be deleting it now")
sh('sudo rm -rf ./influxdb')
if prev_version < StrictVersion('0.3.0'):
# Splitting compose configuration between docker-compose and docker-compose.shared.yml
# Version pinning (0.2.2) will happen automatically
utils.info('Moving system services to docker-compose.shared.yml...')
config = utils.read_compose()
sys_names = [
'mdns',
'eventbus',
'influx',
'datastore',
'history',
'ui',
'traefik',
]
usr_config = {
'version': config['version'],
'services': {
key: svc
for (key, svc) in config['services'].items()
if key not in sys_names
}
}
utils.write_compose(usr_config)
if prev_version < StrictVersion('0.6.0'):
# The datastore service is gone
# Older services may still rely on it
utils.info('Removing `depends_on` fields from docker-compose.yml...')
config = utils.read_compose()
for svc in config['services'].values():
with suppress(KeyError):
del svc['depends_on']
utils.write_compose(config)
# Init dir. It will be filled during upped_migrate
utils.info('Creating redis/ dir...')
sh('mkdir -p redis/')
utils.info('Checking .env variables...')
for (key, default_value) in const.ENV_DEFAULTS.items():
current_value = utils.getenv(key)
if current_value is None:
utils.setenv(key, default_value)
def migrate_compose_datastore():
# The couchdb datastore service is gone
# Older services may still rely on it
utils.info('Removing `depends_on` fields from docker-compose.yml...')
config = utils.read_compose()
for svc in config['services'].values():
with suppress(KeyError):
del svc['depends_on']
utils.write_compose(config)
# Init dir. It will be filled during upped_migrate
utils.info('Creating redis/ dir...')
sh('mkdir -p redis/')
def run_particle_flasher(release: str, pull: bool, cmd: str):
tag = utils.docker_tag(release)
sudo = utils.optsudo()
opts = ' '.join([
'-it',
'--rm',
'--privileged',
'-v /dev:/dev',
'--pull ' + ('always' if pull else 'missing'),
])
sh(f'{sudo}docker-compose --log-level CRITICAL down', check=False)
sh(f'{sudo}docker run {opts} brewblox/firmware-flasher:{tag} {cmd}')
def discover_device(discovery, release):
sudo = utils.optsudo()
mdns = 'brewblox/brewblox-mdns:{}'.format(utils.docker_tag(release))
utils.info('Pulling image...')
sh('{}docker pull {}'.format(sudo, mdns), silent=True)
utils.info('Discovering devices...')
raw_devs = sh('{}docker run '.format(sudo) + '--rm -it ' + '--net=host ' +
'-v /dev/serial:/dev/serial ' +
'{} --cli --discovery {}'.format(mdns, discovery),
capture=True)
return [dev for dev in raw_devs.split('\n') if dev.rstrip()]
def run_esp_flasher(release: str, pull: bool):
tag = utils.docker_tag(release)
sudo = utils.optsudo()
opts = ' '.join([
'-it',
'--rm',
'--privileged',
'-v /dev:/dev',
'-w /app/firmware',
'--entrypoint bash',
'--pull ' + ('always' if pull else 'missing'),
])
sh(f'{sudo}docker-compose --log-level CRITICAL down', check=False)
sh(f'{sudo}docker run {opts} brewblox/brewblox-devcon-spark:{tag} flash')
def makecert(dir, release: str = None):
absdir = Path(dir).resolve()
sudo = utils.optsudo()
tag = utils.docker_tag(release)
sh(f'mkdir -p "{absdir}"')
sh(f'{sudo}docker run' + ' --rm --privileged' + ' --pull always' +
f' -v "{absdir}":/certs/' + f' brewblox/omgwtfssl:{tag}')
sh(f'sudo chmod 644 "{absdir}/brewblox.crt"')
sh(f'sudo chmod 600 "{absdir}/brewblox.key"')
def downed_migrate(prev_version):
"""Migration commands to be executed without any running services"""
if prev_version < StrictVersion('0.2.0'):
# Breaking changes: Influx downsampling model overhaul
# Old data is completely incompatible
utils.select(
'Upgrading to version >=0.2.0 requires a complete reset of your history data. '
+ "We'll be deleting it now")
sh('sudo rm -rf ./influxdb')
if prev_version < StrictVersion('0.3.0'):
# Splitting compose configuration between docker-compose and docker-compose.shared.yml
# Version pinning (0.2.2) will happen automatically
utils.info('Moving system services to docker-compose.shared.yml')
config = utils.read_compose()
sys_names = [
'mdns',
'eventbus',
'influx',
'datastore',
'history',
'ui',
'traefik',
]
usr_config = {
'version': config['version'],
'services': {
key: svc
for (key, svc) in config['services'].items()
if key not in sys_names
}
}
utils.write_compose(usr_config)
utils.info('Writing env values for all variables')
for key in [
const.COMPOSE_FILES_KEY,
const.RELEASE_KEY,
const.HTTP_PORT_KEY,
const.HTTPS_PORT_KEY,
const.MDNS_PORT_KEY,
]:
utils.setenv(key, utils.getenv(key, const.ENV_DEFAULTS[key]))
def add_tilt(force):
"""
Create a service for the Tilt hydrometer.
The service listens for Bluetooth status updates from the Tilt,
and requires the host to have a Bluetooth receiver.
The empty ./tilt dir is created to hold calibration files.
"""
utils.check_config()
utils.confirm_mode()
name = 'tilt'
sudo = utils.optsudo()
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/brewblox-tilt:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'privileged': True,
'network_mode': 'host',
'volumes': [{
'type': 'bind',
'source': f'./{name}',
'target': '/share',
}],
'labels': [
'traefik.enable=false',
],
}
sh(f'mkdir -p ./{name}')
utils.write_compose(config)
click.echo(f'Added Tilt service `{name}`.')
click.echo('It will automatically show up in the UI.\n')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
def edit_avahi_config():
conf = Path(const.AVAHI_CONF)
if not conf.exists():
return
config = ConfigObj(str(conf), file_error=True)
copy = deepcopy(config)
config.setdefault('server', {}).setdefault('use-ipv6', 'no')
config.setdefault('publish', {}).setdefault('publish-aaaa-on-ipv4', 'no')
config.setdefault('reflector', {}).setdefault('enable-reflector', 'yes')
if config == copy:
return
utils.show_data(conf, config.dict())
with NamedTemporaryFile('w') as tmp:
config.filename = None
lines = config.write()
# avahi-daemon.conf requires a 'key=value' syntax
tmp.write('\n'.join(lines).replace(' = ', '=') + '\n')
tmp.flush()
sh(f'sudo chmod --reference={conf} {tmp.name}')
sh(f'sudo cp -fp {tmp.name} {conf}')
if utils.command_exists('systemctl'):
utils.info('Restarting avahi-daemon service...')
sh('sudo systemctl restart avahi-daemon')
else:
utils.warn(
'"systemctl" command not found. Please restart your machine to enable Wifi discovery.'
)
def disable_ssh_accept_env():
"""Disable the 'AcceptEnv LANG LC_*' setting in sshd_config
This setting is default on the Raspberry Pi,
but leads to locale errors when an unsupported LANG is sent.
Given that the Pi by default only includes the en_GB locale,
the chances of being sent a unsupported locale are very real.
"""
file = Path('/etc/ssh/sshd_config')
if not file.exists():
return
content = file.read_text()
updated = re.sub(r'^AcceptEnv LANG LC',
'#AcceptEnv LANG LC',
content,
flags=re.MULTILINE)
if content == updated:
return
with NamedTemporaryFile('w') as tmp:
tmp.write(updated)
tmp.flush()
utils.info('Updating SSHD config to disable AcceptEnv...')
utils.show_data('/etc/ssh/sshd_config', updated)
sh(f'sudo chmod --reference={file} {tmp.name}')
sh(f'sudo cp -fp {tmp.name} {file}')
if utils.command_exists('systemctl'):
utils.info('Restarting SSH service...')
sh('sudo systemctl restart ssh')
def deploy_ctl_wrapper():
sh(f'chmod +x "{const.SCRIPT_DIR}/brewblox-ctl"')
if utils.user_home_exists():
sh(f'mkdir -p "$HOME/.local/bin" && cp "{const.SCRIPT_DIR}/brewblox-ctl" "$HOME/.local/bin/"'
)
else:
sh(f'sudo cp "{const.SCRIPT_DIR}/brewblox-ctl" /usr/local/bin/')
def add_node_red(force):
"""
Create a service for Node-RED.
"""
utils.check_config()
utils.confirm_mode()
name = 'node-red'
sudo = utils.optsudo()
host = utils.host_ip()
port = utils.getenv(const.HTTPS_PORT_KEY)
config = utils.read_compose()
if not force:
check_duplicate(config, name)
config['services'][name] = {
'image': 'brewblox/node-red:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'volumes': [{
'type': 'bind',
'source': f'./{name}',
'target': '/data',
}]
}
sh(f'mkdir -p ./{name}')
if [getgid(), getuid()] != [1000, 1000]:
sh(f'sudo chown -R 1000:1000 ./{name}')
utils.write_compose(config)
click.echo(f'Added Node-RED service `{name}`.')
if utils.confirm('Do you want to run `brewblox-ctl up` now?'):
sh(f'{sudo}docker-compose up -d')
click.echo(f'Visit https://{host}:{port}/{name} in your browser to load the editor.')
def add_particle_udev_rules():
rules_dir = '/etc/udev/rules.d'
target = f'{rules_dir}/50-particle.rules'
if not utils.path_exists(target) and utils.command_exists('udevadm'):
utils.info('Adding udev rules for Particle devices...')
sh(f'sudo mkdir -p {rules_dir}')
sh(f'sudo cp {const.CONFIG_DIR}/50-particle.rules {target}')
sh('sudo udevadm control --reload-rules && sudo udevadm trigger')
def add_plaato(name, token, force):
"""
Create a service for the Plaato airlock.
This will periodically query the Plaato server for current state.
An authentication token is required.
See https://plaato.io/apps/help-center#!hc-auth-token on how to get one.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
config = utils.read_compose()
if name in config['services'] and not force:
click.echo(
'Service "{}" already exists. Use the --force flag if you want to overwrite it'
.format(name))
raise SystemExit(1)
config['services'][name] = {
'image': 'brewblox/brewblox-plaato:${BREWBLOX_RELEASE}',
'restart': 'unless-stopped',
'environment': {
'PLAATO_AUTH': token,
},
'command': '--name=' + name,
}
utils.write_compose(config)
click.echo("Added Plaato service '{}'.".format(name))
click.echo(
'This service publishes history data, but does not have a UI component.'
)
if utils.confirm("Do you want to run 'brewblox-ctl up' now?"):
sh('{}docker-compose up -d --remove-orphans'.format(sudo))
def makecert(dir, release):
"""Generate a self-signed SSL certificate.
\b
Steps:
- Create directory if it does not exist.
- Create brewblox.crt and brewblox.key files.
"""
utils.confirm_mode()
sudo = utils.optsudo()
tag = utils.docker_tag(release)
absdir = Path(dir).absolute()
sh(f'mkdir -p "{absdir}"')
sh(f'{sudo}docker run' + ' --rm --privileged' + ' --pull always' +
f' -v "{absdir}":/certs/' + f' brewblox/omgwtfssl:{tag}')
sh(f'sudo chmod 644 "{absdir}/brewblox.crt"')
sh(f'sudo chmod 600 "{absdir}/brewblox.key"')
def _influx_line_count(service: str, args: str) -> Optional[int]:
sudo = utils.optsudo()
measurement = f'"brewblox"."downsample_1m"."{service}"'
points_field = '"m_ Combined Influx points"'
json_result = sh(
f'{sudo}docker exec influxdb-migrate influx '
'-database brewblox '
f"-execute 'SELECT count({points_field}) FROM {measurement} {args}' "
'-format json',
capture=True)
result = json.loads(json_result)
try:
return result['results'][0]['series'][0]['values'][0][1]
except (IndexError, KeyError):
return None
def fix_ipv6(config_file=None, restart=True):
utils.info('Fixing Docker IPv6 settings...')
if utils.is_wsl():
utils.info('WSL environment detected. Skipping IPv6 config changes.')
return
# Config is either provided, or parsed from active daemon process
if not config_file:
default_config_file = '/etc/docker/daemon.json'
dockerd_proc = sh('ps aux | grep dockerd', capture=True)
proc_match = re.match(r'.*--config-file[\s=](?P<file>.*\.json).*',
dockerd_proc,
flags=re.MULTILINE)
config_file = proc_match and proc_match.group(
'file') or default_config_file
utils.info(f'Using Docker config file {config_file}')
# Read config. Create file if not exists
sh(f"sudo touch '{config_file}'")
config = sh(f"sudo cat '{config_file}'", capture=True)
if 'fixed-cidr-v6' in config:
utils.info('IPv6 settings are already present. Making no changes.')
return
# Edit and write. Do not overwrite existing values
config = json.loads(config or '{}')
config.setdefault('ipv6', False)
config.setdefault('fixed-cidr-v6', '2001:db8:1::/64')
config_str = json.dumps(config, indent=2)
sh(f"echo '{config_str}' | sudo tee '{config_file}' > /dev/null")
# Restart daemon
if restart:
if utils.command_exists('service'):
utils.info('Restarting Docker service...')
sh('sudo service docker restart')
else:
utils.warn(
'"service" command not found. Please restart your machine to apply config changes.'
)