def force_sync():
stop_containers()
parallel([
'peg sshcmd-cluster {} "rm -rf ~/heart_watch"'.format(cluster)
for cluster in CLUSTERS
])
sync()
def stage3():
answer = input(
'\nAre you running in large scale manner (more than 5 machines in total)? [y/n] '
)
if answer == 'y':
answer = input(
'\nHave you followed the "A caveat for running in large scale" section in README? (i.e. you are using my public AMI) [y/n] '
)
if answer != 'y':
logger.warning('Please follow its instructions before proceeding.')
exit(0)
else:
cmds = [
'cd ~/heart_watch && pip3 install -r requirements.txt',
'cd ~/heart_watch && wget http://apache.mirrors.pair.com/kafka/2.2.0/kafka_2.12-2.2.0.tgz',
'cd ~/heart_watch && tar -xzf kafka_2.12-2.2.0.tgz && mv kafka_2.12-2.2.0 kafka',
]
else:
cmds = [
'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -',
'sudo add-apt-repository \'deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable\'',
'sudo add-apt-repository --remove -y ppa:andrei-pozolotin/maven3',
'sudo apt-get -y update',
'apt-cache policy docker-ce',
'sudo kill -9 $(ps aux | grep \'dpkg\' | awk \'{print $2}\')',
'sudo kill -9 $(ps aux | grep \'apt\' | awk \'{print $2}\')',
'sudo killall -r dpkg',
'sudo killall -r apt',
'sudo dpkg --configure -a',
'sudo apt-get install -y docker-ce python3-pip libpq-dev python-dev maven awscli',
'sudo usermod -aG docker ubuntu',
'cd ~/heart_watch && pip3 install -r requirements.txt',
'cd ~/ && wget http://apache.mirrors.pair.com/kafka/2.2.0/kafka_2.12-2.2.0.tgz',
'cd ~/ && tar -xzf kafka_2.12-2.2.0.tgz && mv kafka_2.12-2.2.0 kafka',
'unzip confluent-3.0.0-2.11.zip && mv confluent-3.0.0 confluent',
'cd ~/heart_watch && pip3 install -r requirements.txt',
]
for cmd in cmds:
parallel([
'peg sshcmd-cluster {} "{}"'.format(key, cmd) for key in CLUSTERS
])
logger.info("""
If you responded that you are NOT running in large scale manner, please manually ssh to each machine and run the following commands
sudo curl -L https://github.com/docker/compose/releases/download/1.24.1/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
Then log out and log back again. Then make sure than you can run docker-compose. If you can't, please manually fix it.
""")
def fetch():
parallel(['peg fetch {}'.format(key) for key in CLUSTERS], prompt=False)
def clean_logs():
parallel([
'peg sshcmd-cluster {} "sudo sh -c \'truncate -s 0 /var/lib/docker/containers/*/*-json.log\'"'
.format(key) for key in CLUSTERS
],
prompt=False)
def stage1():
input(
'\nOpen pegasus/{}.yml and adjust the settings. Press ENTER when done: '
.format(', '.join([key for key in CLUSTERS])))
parallel(['peg up pegasus/{}.yml'.format(key) for key in CLUSTERS])
fetch()
def env_override():
def key_val_str(key, val):
return '{}={}\n'.format(key, val)
cluster_servers = get_cluster_servers(force=False)
ksqls_str = ','.join(
['http://{}:8088'.format(s) for s in cluster_servers['ksqls']])
brokers_str = ','.join(
['{}:9092'.format(s) for s in cluster_servers['brokers']])
# TODO: using multiple zookeepers needs extra distribucted setup
# zookeepers_str = ','.join(['{}:2181'.format(s) for s in cluster_servers['brokers']])
zookeepers_str = cluster_servers['brokers'][0] + ':2181'
schema_registry_kafkastore_bootstrap_servers = ','.join(
['PLAINTEXT://{}:9092'.format(s) for s in cluster_servers['brokers']])
master_dns = cluster_servers['brokers'][0]
for cluster, servers in cluster_servers.items():
for i, server in enumerate(servers):
local_path = '.env.override_{}_{}'.format('brokers', i + 1)
with open('.env.override/zookeeper', 'w') as f:
for j in range(len(servers)):
f.write(
key_val_str(
'ZOOKEEPER_SERVER_{}'.format(j + 1),
'{}:2888:3888'.format(servers[j]),
))
with open('.env.override/broker', 'w') as f:
f.write(
key_val_str('KAFKA_ADVERTISED_LISTENERS',
'PLAINTEXT://{}:9092'.format(server)), )
f.write(key_val_str('KAFKA_BROKER_ID', i + 1))
# f.write(key_val_str('KAFKA_ZOOKEEPER_CONNECT', "{}".format(zookeepers_str)))
f.write(
key_val_str('CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS',
brokers_str))
f.write(
key_val_str('CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT',
zookeepers_str))
with open('.env.override/schema-registry', 'w') as f:
f.write(key_val_str('SCHEMA_REGISTRY_HOST_NAME', master_dns))
f.write(
key_val_str('SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL',
zookeepers_str))
with open('.env.override/connect', 'w') as f:
f.write(
key_val_str('CONNECT_BOOTSTRAP_SERVERS',
'{}:9092'.format(master_dns)))
f.write(
key_val_str('CONNECT_REST_ADVERTISED_HOST_NAME',
master_dns))
f.write(
key_val_str('CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL',
'http://{}:8081'.format(master_dns)))
with open('.env.override/rest-proxy', 'w') as f:
f.write(
key_val_str('KAFKA_REST_BOOTSTRAP_SERVERS', brokers_str))
f.write(
key_val_str('KAFKA_REST_SCHEMA_REGISTRY_URL',
'http://{}:8081'.format(master_dns)))
with open('.env.override/control-center', 'w') as f:
f.write(
key_val_str('CONTROL_CENTER_BOOTSTRAP_SERVERS',
brokers_str))
f.write(
key_val_str('CONTROL_CENTER_ZOOKEEPER_CONNECT',
zookeepers_str))
f.write(key_val_str('CONTROL_CENTER_KSQL_URL', ksqls_str))
f.write(
key_val_str('CONTROL_CENTER_KSQL_ADVERTISED_URL',
'{}:8088'.format(master_dns)))
f.write(
key_val_str('CONTROL_CENTER_SCHEMA_REGISTRY_URL',
'http://{}:8081'.format(master_dns)))
with open('.env.override/ksql-server', 'w') as f:
f.write(key_val_str('KSQL_BOOTSTRAP_SERVERS', brokers_str))
f.write(key_val_str('KSQL_HOST', server))
f.write(
key_val_str('KSQL_KSQL_SCHEMA_REGISTRY_URL',
'http://{}:8081'.format(master_dns)))
f.write(key_val_str('KSQL_HOST_NAME', server))
services = [
'zookeeper', 'broker', 'schema-registry', 'connect',
'rest-proxy', 'control-center', 'ksql-server'
]
cmds = []
for service in services:
cmds.append(
'peg scp from-local {cluster} {node} {local_path} {remove_path} && rm {local_path}'
.format(
cluster=cluster,
node=str(i + 1),
local_path=join('.env.override', service),
remove_path=join(REMOTE_ROOT_DIR, '.env.override',
service),
))
parallel(cmds, prompt=False)