def check_ports():
if utils.path_exists('./docker-compose.yml'):
utils.info('Stopping services...')
sh('{}docker-compose down --remove-orphans'.format(utils.optsudo()))
ports = [
utils.getenv(key, const.ENV_DEFAULTS[key]) for key in [
const.HTTP_PORT_KEY,
const.HTTPS_PORT_KEY,
const.MDNS_PORT_KEY,
]
]
utils.info('Checking ports...')
retv = sh('sudo netstat -tulpn', capture=True)
lines = retv.split('\n')
used_ports = []
used_lines = []
for port in ports:
for line in lines:
if re.match(r'.*(:::|0.0.0.0:){}\s.*'.format(port), line):
used_ports.append(port)
used_lines.append(line)
break
if used_ports:
utils.warn(
'Port(s) {} already in use. '.format(', '.join(used_ports)) +
"Run 'brewblox-ctl service ports' to configure Brewblox ports.")
for line in used_lines:
utils.warn(line)
if not utils.confirm('Do you want to continue?'):
raise SystemExit(1)
def migrate_couchdb():
urllib3.disable_warnings()
sudo = utils.optsudo()
opts = utils.ctx_opts()
redis_url = utils.datastore_url()
couch_url = 'http://localhost:5984'
utils.info('Migrating datastore from CouchDB to Redis...')
if opts.dry_run:
utils.info('Dry run. Skipping migration...')
return
if not utils.path_exists('./couchdb/'):
utils.info('couchdb/ dir not found. Skipping migration...')
return
utils.info('Starting a temporary CouchDB container on port 5984...')
sh(f'{sudo}docker rm -f couchdb-migrate', check=False)
sh(f'{sudo}docker run --rm -d'
' --name couchdb-migrate'
' -v "$(pwd)/couchdb/:/opt/couchdb/data/"'
' -p "5984:5984"'
' treehouses/couchdb:2.3.1')
sh(f'{const.CLI} http wait {couch_url}')
sh(f'{const.CLI} http wait {redis_url}/ping')
resp = requests.get(f'{couch_url}/_all_dbs')
resp.raise_for_status()
dbs = resp.json()
for db in ['brewblox-ui-store', 'brewblox-automation']:
if db in dbs:
resp = requests.get(f'{couch_url}/{db}/_all_docs',
params={'include_docs': True})
resp.raise_for_status()
docs = [v['doc'] for v in resp.json()['rows']]
# Drop invalid names
docs[:] = [d for d in docs if len(d['_id'].split('__', 1)) == 2]
for d in docs:
segments = d['_id'].split('__', 1)
d['namespace'] = f'{db}:{segments[0]}'
d['id'] = segments[1]
del d['_rev']
del d['_id']
resp = requests.post(f'{redis_url}/mset',
json={'values': docs},
verify=False)
resp.raise_for_status()
utils.info(f'Migrated {len(docs)} entries from {db}')
if 'spark-service' in dbs:
resp = requests.get(f'{couch_url}/spark-service/_all_docs',
params={'include_docs': True})
resp.raise_for_status()
docs = [v['doc'] for v in resp.json()['rows']]
for d in docs:
d['namespace'] = 'spark-service'
d['id'] = d['_id']
del d['_rev']
del d['_id']
resp = requests.post(f'{redis_url}/mset',
json={'values': docs},
verify=False)
resp.raise_for_status()
utils.info(f'Migrated {len(docs)} entries from spark-service')
sh(f'{sudo}docker stop couchdb-migrate')
sh('sudo mv couchdb/ couchdb-migrated-' +
datetime.now().strftime('%Y%m%d'))
def migrate_influxdb(
target: str = 'victoria',
duration: str = '',
services: List[str] = [],
offsets: List[Tuple[str, int]] = [],
):
"""Exports InfluxDB history data.
The exported data is either immediately imported to the new history database,
or saved to file.
"""
opts = utils.ctx_opts()
sudo = utils.optsudo()
date = datetime.now().strftime('%Y%m%d_%H%M')
utils.warn('Depending on the amount of data, this may take some hours.')
utils.warn(
'You can use your system as normal while the migration is in progress.'
)
utils.warn('The migration can safely be stopped and restarted or resumed.')
utils.warn(
'For more info, see https://brewblox.netlify.app/dev/migration/influxdb.html'
)
if opts.dry_run:
utils.info('Dry run. Skipping migration...')
return
if not utils.path_exists('./influxdb/'):
utils.info('influxdb/ dir not found. Skipping migration...')
return
utils.info('Starting InfluxDB container...')
# Stop container in case previous migration was cancelled
sh(f'{sudo}docker stop influxdb-migrate > /dev/null', check=False)
# Start standalone container
# We'll communicate using 'docker exec', so no need to publish a port
sh(f'{sudo}docker run '
'--rm -d '
'--name influxdb-migrate '
'-v "$(pwd)/influxdb:/var/lib/influxdb" '
'influxdb:1.8 '
'> /dev/null')
# Do a health check until startup is done
inner_cmd = 'curl --output /dev/null --silent --fail http://localhost:8086/health'
bash_cmd = f'until $({inner_cmd}); do sleep 1 ; done'
sh(f"{sudo}docker exec influxdb-migrate bash -c '{bash_cmd}'")
# Determine relevant measurement
# Export all of them if not specified by user
if not services:
services = _influx_measurements()
utils.info(f'Exporting services: {", ".join(services)}')
# Export data and import to target
for svc in services:
offset = next((v for v in offsets if v[0] == svc), ('default', 0))[1]
_copy_influx_measurement(svc, date, duration, target, offset)
# Stop migration container
sh(f'{sudo}docker stop influxdb-migrate > /dev/null', check=False)
def setup(port_check):
"""Run first-time setup in Brewblox directory.
Run after brewblox-ctl install, in the newly created Brewblox directory.
This will create all required configuration files for your system.
You can safely use this command to partially reset your system.
Before making any changes, it will check for existing files,
and prompt if any are found. It will do so separately for docker-compose,
datastore, history, and gateway files.
Choose to skip any, and the others will still be created and configured.
\b
Steps:
- Check whether files already exist.
- Set .env values.
- Create docker-compose configuration files. (Optional)
- Pull docker images.
- Create datastore (CouchDB) directory. (Optional)
- Create history (InfluxDB) directory. (Optional)
- Create gateway (Traefik) directory. (Optional)
- Create SSL certificates. (Optional)
- Start and configure services. (Optional)
- Stop all services.
- Set version number in .env.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
datastore_url = utils.datastore_url()
history_url = utils.history_url()
upped_services = ['traefik', 'influx', 'history']
preset_modules = ['services', 'dashboards', 'dashboard-items']
if port_check:
check_ports()
skip_compose = \
utils.path_exists('./docker-compose.yml') \
and utils.confirm('This directory already contains a docker-compose.yml file. ' +
'Do you want to keep it?')
skip_datastore = \
utils.path_exists('./couchdb/') \
and utils.confirm('This directory already contains Couchdb datastore files. ' +
'Do you want to keep them?')
skip_history = \
utils.path_exists('./influxdb/') \
and utils.confirm('This directory already contains Influx history files. ' +
'Do you want to keep them?')
skip_gateway = \
utils.path_exists('./traefik/') \
and utils.confirm('This directory already contains Traefik gateway files. ' +
'Do you want to keep them?')
utils.info('Setting .env values...')
for key, default_val in const.ENV_DEFAULTS.items():
utils.setenv(key, utils.getenv(key, default_val))
if not skip_compose:
utils.info('Copying configuration...')
sh('cp -f {}/* ./'.format(const.CONFIG_DIR))
# Stop and pull after we're sure we have a compose file
utils.info('Stopping services...')
sh('{}docker-compose down --remove-orphans'.format(sudo))
utils.info('Pulling docker images...')
sh('{}docker-compose pull'.format(sudo))
if not skip_datastore:
utils.info('Creating datastore directory...')
upped_services.append('datastore')
sh('sudo rm -rf ./couchdb/; mkdir ./couchdb/')
if not skip_history:
utils.info('Creating history directory...')
sh('sudo rm -rf ./influxdb/; mkdir ./influxdb/')
if not skip_gateway:
utils.info('Creating gateway directory...')
sh('sudo rm -rf ./traefik/; mkdir ./traefik/')
utils.info('Creating SSL certificate...')
sh('{}docker run --rm -v "$(pwd)"/traefik/:/certs/ '.format(sudo) +
'brewblox/omgwtfssl:{}'.format(utils.docker_tag()))
sh('sudo chmod 644 traefik/brewblox.crt')
sh('sudo chmod 600 traefik/brewblox.key')
# Bring images online that we will send configuration
utils.info('Starting configured services...')
sh('{}docker-compose up -d --remove-orphans {}'.format(
sudo, ' '.join(upped_services)))
if not skip_datastore:
# Generic datastore setup
utils.info('Configuring datastore settings...')
sh('{} http wait {}'.format(const.CLI, datastore_url))
sh('{} http put {}/_users'.format(const.CLI, datastore_url))
sh('{} http put {}/_replicator'.format(const.CLI, datastore_url))
sh('{} http put {}/_global_changes'.format(const.CLI, datastore_url))
sh('{} http put {}/{}'.format(const.CLI, datastore_url,
const.UI_DATABASE))
# Load presets
utils.info('Loading preset data...')
for mod in preset_modules:
sh('{} http post {}/{}/_bulk_docs -f {}/{}.json'.format(
const.CLI, datastore_url, const.UI_DATABASE, const.PRESETS_DIR,
mod))
# Always setup history
utils.info('Configuring history settings...')
sh('{} http wait {}/ping'.format(const.CLI, history_url))
sh('{} http post {}/query/configure'.format(const.CLI, history_url))
# Setup is done - leave system in stable state
utils.info('Stopping services...')
sh('{}docker-compose down'.format(utils.optsudo()))
# Setup is complete and ok - now set CFG version
utils.setenv(const.CFG_VERSION_KEY, const.CURRENT_VERSION)
utils.info('All done!')
def setup(ctx, avahi_config, pull, port_check):
"""Run first-time setup in Brewblox directory.
Run after brewblox-ctl install, in the newly created Brewblox directory.
This will create all required configuration files for your system.
You can safely use this command to partially reset your system.
Before making any changes, it will check for existing files,
and prompt if any are found. It will do so separately for docker-compose,
datastore, history, and gateway files.
Choose to skip any, and the others will still be created and configured.
\b
Steps:
- Check whether files already exist.
- Set .env values.
- Update avahi-daemon config. (Optional)
- Create docker-compose configuration files. (Optional)
- Pull docker images. (Optional)
- Create datastore (Redis) directory. (Optional)
- Create history (Victoria) directory. (Optional)
- Create gateway (Traefik) directory. (Optional)
- Create SSL certificates. (Optional)
- Start and configure services. (Optional)
- Stop all services.
- Set version number in .env.
"""
utils.check_config()
utils.confirm_mode()
sudo = utils.optsudo()
if port_check:
check_ports()
skip_compose = \
utils.path_exists('./docker-compose.yml') \
and utils.confirm('This directory already contains a docker-compose.yml file. ' +
'Do you want to keep it?')
skip_datastore = \
utils.path_exists('./redis/') \
and utils.confirm('This directory already contains Redis datastore files. ' +
'Do you want to keep them?')
skip_history = \
utils.path_exists('./victoria/') \
and utils.confirm('This directory already contains Victoria history files. ' +
'Do you want to keep them?')
skip_gateway = \
utils.path_exists('./traefik/') \
and utils.confirm('This directory already contains Traefik gateway files. ' +
'Do you want to keep them?')
skip_eventbus = \
utils.path_exists('./mosquitto/') \
and utils.confirm('This directory already contains Mosquitto config files. ' +
'Do you want to keep them?')
utils.info('Setting .env values...')
for key, default_val in const.ENV_DEFAULTS.items():
utils.setenv(key, utils.getenv(key, default_val))
if avahi_config:
utils.update_avahi_config()
utils.info('Copying docker-compose.shared.yml...')
sh(f'cp -f {const.CONFIG_DIR}/docker-compose.shared.yml ./')
if not skip_compose:
utils.info('Copying docker-compose.yml...')
sh(f'cp -f {const.CONFIG_DIR}/docker-compose.yml ./')
# Stop and pull after we're sure we have a compose file
utils.info('Stopping services...')
sh(f'{sudo}docker-compose down')
if pull:
utils.info('Pulling docker images...')
sh(f'{sudo}docker-compose pull')
if not skip_datastore:
utils.info('Creating datastore directory...')
sh('sudo rm -rf ./redis/; mkdir ./redis/')
if not skip_history:
utils.info('Creating history directory...')
sh('sudo rm -rf ./victoria/; mkdir ./victoria/')
if not skip_gateway:
utils.info('Creating gateway directory...')
sh('sudo rm -rf ./traefik/; mkdir ./traefik/')
utils.info('Creating SSL certificate...')
ctx.invoke(makecert)
if not skip_eventbus:
utils.info('Creating mosquitto config directory...')
sh('sudo rm -rf ./mosquitto/; mkdir ./mosquitto/')
# Always copy cert config to traefik dir
sh(f'cp -f {const.CONFIG_DIR}/traefik-cert.yaml ./traefik/')
# Setup is complete and ok - now set CFG version
utils.setenv(const.CFG_VERSION_KEY, const.CURRENT_VERSION)
utils.info('All done!')