def test_path_exists(mocked_ext):
mocked_ext['path'].exists.side_effect = [
True,
False,
]
assert utils.path_exists('p1')
assert not utils.path_exists('p2')
def check_init_opts(self):
self.init_compose = True
self.init_datastore = True
self.init_history = True
self.init_gateway = True
self.init_eventbus = True
if utils.path_exists('./docker-compose.yml'):
self.init_compose = not utils.confirm(
'This directory already contains a docker-compose.yml file. ' +
'Do you want to keep it?')
if utils.path_exists('./redis/'):
self.init_datastore = not utils.confirm(
'This directory already contains Redis datastore files. ' +
'Do you want to keep them?')
if utils.path_exists('./victoria/'):
self.init_history = not utils.confirm(
'This directory already contains Victoria history files. ' +
'Do you want to keep them?')
if utils.path_exists('./traefik/'):
self.init_gateway = not utils.confirm(
'This directory already contains Traefik gateway files. ' +
'Do you want to keep them?')
if utils.path_exists('./mosquitto/'):
self.init_eventbus = not utils.confirm(
'This directory already contains Mosquitto config files. ' +
'Do you want to keep them?')
def test_path_exists(mocker):
m_path = mocker.patch(TESTED + '.Path').return_value
m_path.exists.side_effect = [
True,
False,
]
assert utils.path_exists('p1')
assert not utils.path_exists('p2')
def check_lib():
if utils.is_brewblox_cwd() \
and not utils.path_exists('./brewblox_ctl_lib/__init__.py') \
and utils.confirm(
'brewblox-ctl requires extensions that match your Brewblox release. ' +
'Do you want to download them now?'):
utils.load_ctl_lib(utils.ContextOpts(dry_run=False, verbose=True))
def check_ports():
if utils.path_exists('./docker-compose.yml'):
utils.info('Stopping services...')
sh(f'{utils.optsudo()}docker-compose down')
ports = [
int(utils.getenv(key, const.ENV_DEFAULTS[key])) for key in [
const.HTTP_PORT_KEY,
const.HTTPS_PORT_KEY,
const.MQTT_PORT_KEY,
]
]
try:
port_connnections = [
conn for conn in psutil.net_connections()
if conn.laddr.ip in ['::', '0.0.0.0'] and conn.laddr.port in ports
]
except psutil.AccessDenied:
utils.warn(
'Unable to read network connections. You need to run `netstat` or `lsof` manually.'
)
port_connnections = []
if port_connnections:
port_str = ', '.join(
set(str(conn.laddr.port) for conn in port_connnections))
utils.warn(f'Port(s) {port_str} already in use.')
utils.warn(
'Run `brewblox-ctl service ports` to configure Brewblox ports.')
if not utils.confirm('Do you want to continue?'):
raise SystemExit(1)
def install_ctl_package(download: str = 'always'): # always | missing | never
exists = utils.path_exists('./brewblox-ctl.tar.gz')
release = utils.getenv(const.CTL_RELEASE_KEY) or utils.getenv(
const.RELEASE_KEY)
if download == 'always' or download == 'missing' and not exists:
sh(f'wget -q -O ./brewblox-ctl.tar.gz https://brewblox.blob.core.windows.net/ctl/{release}/brewblox-ctl.tar.gz'
)
sh('python3 -m pip install ./brewblox-ctl.tar.gz')
def add_particle_udev_rules():
rules_dir = '/etc/udev/rules.d'
target = f'{rules_dir}/50-particle.rules'
if not utils.path_exists(target) and utils.command_exists('udevadm'):
utils.info('Adding udev rules for Particle devices...')
sh(f'sudo mkdir -p {rules_dir}')
sh(f'sudo cp {const.CONFIG_DIR}/50-particle.rules {target}')
sh('sudo udevadm control --reload-rules && sudo udevadm trigger')
def prepare_flasher(release, pull):
tag = utils.docker_tag(release)
sudo = utils.optsudo()
if pull:
utils.info('Pulling flasher image...')
sh('{}docker pull brewblox/firmware-flasher:{}'.format(sudo, tag))
if utils.path_exists('./docker-compose.yml'):
utils.info('Stopping services...')
sh('{}docker-compose down'.format(sudo))
def usage_hint(ex):
if ex.message and 'No such command' in ex.message and not utils.is_brewblox_cwd(
):
default_dir = path.expanduser('~/brewblox')
prompt = [
'',
'Many commands only work if your current directory is a Brewblox directory.',
]
if utils.path_exists('{}/docker-compose.yml'.format(default_dir)):
prompt += [
'It looks like you installed Brewblox in the default location.',
'To navigate there, run:', '', ' cd {}'.format(default_dir),
''
]
click.echo('\n'.join(prompt))
def save(file, force):
"""Save Brewblox directory to snapshot.
This can be used to move Brewblox installations between hosts.
To load the snapshot, use `brewblox-ctl install --snapshot ARCHIVE`
or `brewblox-ctl snapshot load --file ARCHIVE`
Block data stored on Spark controllers is not included in the snapshot.
"""
utils.check_config()
utils.confirm_mode()
dir = Path('./').resolve()
if utils.path_exists(file):
if force or utils.confirm(f'`{file}` already exists. ' +
'Do you want to overwrite it?'):
sh(f'rm -f {file}')
else:
return
sh(f'sudo tar -C {dir.parent} --exclude .venv -czf {file} {dir.name}')
click.echo(Path(file).resolve())
def migrate_couchdb():
urllib3.disable_warnings()
sudo = utils.optsudo()
opts = utils.ctx_opts()
redis_url = utils.datastore_url()
couch_url = 'http://localhost:5984'
utils.info('Migrating datastore from CouchDB to Redis...')
if opts.dry_run:
utils.info('Dry run. Skipping migration...')
return
if not utils.path_exists('./couchdb/'):
utils.info('couchdb/ dir not found. Skipping migration...')
return
utils.info('Starting a temporary CouchDB container on port 5984...')
sh(f'{sudo}docker rm -f couchdb-migrate', check=False)
sh(f'{sudo}docker run --rm -d'
' --name couchdb-migrate'
' -v "$(pwd)/couchdb/:/opt/couchdb/data/"'
' -p "5984:5984"'
' treehouses/couchdb:2.3.1')
sh(f'{const.CLI} http wait {couch_url}')
sh(f'{const.CLI} http wait {redis_url}/ping')
resp = requests.get(f'{couch_url}/_all_dbs')
resp.raise_for_status()
dbs = resp.json()
for db in ['brewblox-ui-store', 'brewblox-automation']:
if db in dbs:
resp = requests.get(f'{couch_url}/{db}/_all_docs',
params={'include_docs': True})
resp.raise_for_status()
docs = [v['doc'] for v in resp.json()['rows']]
# Drop invalid names
docs[:] = [d for d in docs if len(d['_id'].split('__', 1)) == 2]
for d in docs:
segments = d['_id'].split('__', 1)
d['namespace'] = f'{db}:{segments[0]}'
d['id'] = segments[1]
del d['_rev']
del d['_id']
resp = requests.post(f'{redis_url}/mset',
json={'values': docs},
verify=False)
resp.raise_for_status()
utils.info(f'Migrated {len(docs)} entries from {db}')
if 'spark-service' in dbs:
resp = requests.get(f'{couch_url}/spark-service/_all_docs',
params={'include_docs': True})
resp.raise_for_status()
docs = [v['doc'] for v in resp.json()['rows']]
for d in docs:
d['namespace'] = 'spark-service'
d['id'] = d['_id']
del d['_rev']
del d['_id']
resp = requests.post(f'{redis_url}/mset',
json={'values': docs},
verify=False)
resp.raise_for_status()
utils.info(f'Migrated {len(docs)} entries from spark-service')
sh(f'{sudo}docker stop couchdb-migrate')
sh('sudo mv couchdb/ couchdb-migrated-' +
datetime.now().strftime('%Y%m%d'))
def migrate_influxdb(
target: str = 'victoria',
duration: str = '',
services: List[str] = [],
offsets: List[Tuple[str, int]] = [],
):
"""Exports InfluxDB history data.
The exported data is either immediately imported to the new history database,
or saved to file.
"""
opts = utils.ctx_opts()
sudo = utils.optsudo()
date = datetime.now().strftime('%Y%m%d_%H%M')
utils.warn('Depending on the amount of data, this may take some hours.')
utils.warn(
'You can use your system as normal while the migration is in progress.'
)
utils.warn('The migration can safely be stopped and restarted or resumed.')
utils.warn(
'For more info, see https://brewblox.netlify.app/dev/migration/influxdb.html'
)
if opts.dry_run:
utils.info('Dry run. Skipping migration...')
return
if not utils.path_exists('./influxdb/'):
utils.info('influxdb/ dir not found. Skipping migration...')
return
utils.info('Starting InfluxDB container...')
# Stop container in case previous migration was cancelled
sh(f'{sudo}docker stop influxdb-migrate > /dev/null', check=False)
# Start standalone container
# We'll communicate using 'docker exec', so no need to publish a port
sh(f'{sudo}docker run '
'--rm -d '
'--name influxdb-migrate '
'-v "$(pwd)/influxdb:/var/lib/influxdb" '
'influxdb:1.8 '
'> /dev/null')
# Do a health check until startup is done
inner_cmd = 'curl --output /dev/null --silent --fail http://localhost:8086/health'
bash_cmd = f'until $({inner_cmd}); do sleep 1 ; done'
sh(f"{sudo}docker exec influxdb-migrate bash -c '{bash_cmd}'")
# Determine relevant measurement
# Export all of them if not specified by user
if not services:
services = _influx_measurements()
utils.info(f'Exporting services: {", ".join(services)}')
# Export data and import to target
for svc in services:
offset = next((v for v in offsets if v[0] == svc), ('default', 0))[1]
_copy_influx_measurement(svc, date, duration, target, offset)
# Stop migration container
sh(f'{sudo}docker stop influxdb-migrate > /dev/null', check=False)
def install(use_defaults,
apt_install,
docker_install,
docker_user,
no_reboot,
dir,
release):
"""Create Brewblox directory; install system dependencies; reboot.
Brewblox can be installed multiple times on the same computer.
Settings and databases are stored in a Brewblox directory (default: ./brewblox).
This command also installs system-wide dependencies (docker).
After `brewblox-ctl install`, run `brewblox-ctl setup` in the created Brewblox directory.
A reboot is required after installing docker, or adding the user to the 'docker' group.
By default, `brewblox-ctl install` attempts to download packages using the apt package manager.
If you are using a system without apt (eg. Synology NAS), this step will be skipped.
You will need to manually install any missing libraries.
\b
Steps:
- Install apt packages.
- Install docker.
- Add user to 'docker' group.
- Create Brewblox directory (default ./brewblox).
- Set variables in .env file.
- Reboot.
"""
utils.confirm_mode()
apt_deps = 'curl net-tools libssl-dev libffi-dev'
user = utils.getenv('USER')
default_dir = path.abspath('./brewblox')
prompt_reboot = True
if use_defaults is None:
use_defaults = utils.confirm('Do you want to install with default settings?')
# Check if packages should be installed
if not utils.command_exists('apt'):
utils.info('Apt is not available. You may need to find another way to install dependencies.')
utils.info('Apt packages: "{}"'.format(apt_deps))
apt_install = False
if apt_install is None:
if use_defaults:
apt_install = True
else:
apt_install = utils.confirm('Do you want to install apt packages "{}"?'.format(apt_deps))
# Check if docker should be installed
if utils.command_exists('docker'):
utils.info('Docker is already installed.')
docker_install = False
if docker_install is None:
if use_defaults:
docker_install = True
else:
docker_install = utils.confirm('Do you want to install docker?')
# Check if user should be added to docker group
if utils.is_docker_user():
utils.info('{} already belongs to the docker group.'.format(user))
docker_user = False
if docker_user is None:
if use_defaults:
docker_user = True
else:
docker_user = utils.confirm('Do you want to run docker commands without sudo?')
# Check used directory
if dir is None:
if use_defaults or utils.confirm("The default directory is '{}'. Do you want to continue?".format(default_dir)):
dir = default_dir
else:
return
if utils.path_exists(dir):
if not utils.confirm('{} already exists. Do you want to continue?'.format(dir)):
return
if not no_reboot:
prompt_reboot = utils.confirm('A reboot is required after installation. ' +
'Do you want to be prompted before that happens?')
# Install Apt packages
if apt_install:
utils.info('Installing apt packages...')
sh([
'sudo apt update',
'sudo apt upgrade -y',
'sudo apt install -y {}'.format(apt_deps),
])
else:
utils.info('Skipped: apt install.')
# Install docker
if docker_install:
utils.info('Installing docker...')
sh('curl -sL get.docker.com | sh')
else:
utils.info('Skipped: docker install.')
# Add user to 'docker' group
if docker_user:
utils.info("Adding {} to 'docker' group...".format(user))
sh('sudo usermod -aG docker $USER')
else:
utils.info("Skipped: adding {} to 'docker' group.".format(user))
# Create install directory
utils.info('Creating Brewblox directory ({})...'.format(dir))
sh('mkdir -p {}'.format(dir))
# Set variables in .env file
utils.info('Setting variables in .env file...')
dotenv_path = path.abspath('{}/.env'.format(dir))
sh('touch {}'.format(dotenv_path))
utils.setenv(const.RELEASE_KEY, release, dotenv_path)
utils.setenv(const.CFG_VERSION_KEY, '0.0.0', dotenv_path)
utils.setenv(const.SKIP_CONFIRM_KEY, str(use_defaults), dotenv_path)
utils.info('Done!')
# Reboot
if not no_reboot:
if prompt_reboot:
utils.info('Press ENTER to reboot.')
input()
else:
utils.info('Rebooting in 10 seconds...')
sleep(10)
sh('sudo reboot')
else:
utils.info('Skipped: reboot.')
