def main():
my_basedir = os.path.dirname(os.path.realpath(__file__)).replace(
'/bin', '')
my_util = RegisterUtil("check_kafka", "{}/log".format(my_basedir))
my_region = my_util.exec_shell_command(
"curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | awk -F\\\" '{print $4}'"
)
my_file = RegisterFile(my_util, my_region, my_basedir)
params = my_file.process_conf(
"{}/etc/register_kafka.conf".format(my_basedir))
http_ip = '0.0.0.0'
if 'check_broker_port' in params:
http_port = int(params['check_broker_port'])
else:
http_port = 2180
http_handler = BrokerRequestHandler
my_util.log.info("Starting server at {}:{}".format(http_ip, http_port))
# httpd = SocketServer.TCPServer((http_ip, http_port), http_handler)
httpd = SimpleThreadingServer((http_ip, http_port), http_handler)
httpd.serve_forever()
def main():
my_basedir = os.path.dirname(os.path.realpath(__file__)).replace('/bin', '')
my_util = RegisterUtil("register_zookeeper", "{}/log".format(my_basedir))
my_util.log.info("service_register_zookeeper: starting registration service")
my_region = my_util.exec_shell_command("curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | awk -F\\\" '{print $4}'")
my_file = RegisterFile(my_util, my_region, my_basedir)
params = my_file.process_conf("{}/etc/register_zookeeper.conf".format(my_basedir))
my_file.set_bucket_name(params['s3_bucket'])
while 1:
state = my_file.get_zookeeper_state()
my_util.log.info("service_register_zookeeper: state {}".format(state))
if state == "1":
command_shell = "python {}/bin/register_zookeeper.py".format(my_basedir)
my_util.exec_shell_command(command_shell)
my_util.loop_sleep(120)
class ZookeeperRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
my_basedir = os.path.dirname(os.path.realpath(__file__)).replace(
'/bin', '')
my_util = RegisterUtil("register_zookeeper", "{}/log".format(my_basedir))
check_output = ""
check_iid = ""
check_message = ""
def _set_headers(self):
command1 = "echo ruok | nc localhost 2181"
command2 = "curl -s http://169.254.169.254/latest/meta-data/instance-id"
self.check_output = self.my_util.exec_shell_command(
command1, enable_logging=False)
self.check_iid = self.my_util.exec_shell_command(command2,
enable_logging=False)
self.check_message = "{} {}".format(self.check_output, self.check_iid)
if self.check_output == "imok":
self.my_util.log.info("check_zookeeper: [200] {}".format(
self.check_message))
self.send_response(200)
else:
self.my_util.log.error("check_zookeeper: [500] {}".format(
self.check_message))
self.send_response(500)
self.send_header('Content-Type', 'text/html')
self.send_header('Content-Length', len(self.check_message))
self.end_headers()
def do_GET(self):
self._set_headers()
self.wfile.write(self.check_message)
def do_HEAD(self):
self._set_headers()
self.wfile.write(self.check_message)
def main():
my_basedir = os.path.dirname(os.path.realpath(__file__)).replace(
'/bin', '')
my_util = RegisterUtil("register_zookeeper", "{}/log".format(my_basedir))
my_util.log.info("register_zookeeper: beginning registration process")
my_util.log.info("register_zookeeper: using basedir={}".format(my_basedir))
# my_hostname = my_util.exec_shell_command("curl -s http://169.254.169.254/latest/meta-data/hostname")
my_hostname = my_util.exec_shell_command(
"curl -s http://169.254.169.254/latest/meta-data/local-ipv4")
my_region = my_util.exec_shell_command(
"curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | awk -F\\\" '{{print $4}}'"
)
my_util.log.info("register_zookeeper: hostname {}".format(my_hostname))
my_util.log.info("register_zookeeper: region {}".format(my_region))
my_asg = RegisterASG(my_util, my_region)
asg_name = my_asg.get_asg_name()
asg_desired_cap = my_asg.get_desired_capacity(asg_name)
asg_elb = my_asg.get_asg_elbs(asg_name)
my_util.log.info(
"register_zookeeper: instance is part of asg {}".format(asg_name))
my_util.log.info(
"register_zookeeper: asg desired capacity is currently {}".format(
asg_desired_cap))
my_util.log.info(
"register_zookeeper: asg is behind elb {}".format(asg_elb))
my_file = RegisterFile(my_util, my_region, my_basedir)
params = my_file.process_conf(
"{}/etc/register_zookeeper.conf".format(my_basedir))
my_file.set_bucket_name(params['s3_bucket'])
my_file.set_config_zook(params['cfg_zookeeper'])
my_queue_zook = RegisterQueue(my_util, my_region, params['sqs_url_zook'])
my_queue_replace = RegisterQueue(my_util, my_region,
params['sqs_url_replace'])
firstrun = my_file.get_zookeeper_firstrun()
my_util.log.info("register_zookeeper: firstrun: {}".format(firstrun))
# get register_zookeeper.connection file
hosts_s3 = my_file.get_connection_file()
# if connection file exists, ensemble has already been stood up, thus this launch is a replacement
# therefore, this instance must initialize but also tell other instances to perform replacement
# achieved by setting state flag to 1, service_register_zookeeper.py listens for this state change
# state = 0, brand new cluster build
# state = 1, existing cluster, failure replacement
my_util.log.info(
"register_zookeeper: remote execution check: {}, {}".format(
firstrun, len(hosts_s3)))
if firstrun == '1' and len(hosts_s3) > 0:
my_file.write_zookeeper_state('1')
# send my hostname to the queue
my_queue_zook.send_hostname(my_hostname)
# get online instances that registered to the sqs queue, will wait until param_total hosts are in the queue
hosts_queue = my_queue_zook.get_hostnames(asg_desired_cap)
# determine proper server list identities
hosts_final, hosts_replaced, hosts_existing, hosts_new = my_queue_zook.process_hosts(
hosts_s3, hosts_queue)
my_util.log.info("register_zookeeper: master list: {}".format(hosts_final))
my_util.log.info(
"register_zookeeper: existing hosts: {}".format(hosts_existing))
my_util.log.info(
"register_zookeeper: replaced hosts: {}".format(hosts_replaced))
my_util.log.info("register_zookeeper: new hosts: {}".format(hosts_new))
if firstrun == '1':
# scenario: brand new instantiation
# set firstrun to '0'
my_file.write_zookeeper_firstrun('0')
# write new entries to zookeeper properties
my_file.write_zookeeper_properties(hosts_final)
# get myid from the final list and write myid file
my_file.write_zookeeper_id(hosts_final.index(my_hostname) + 1)
# need to wait for other replacement hosts to complete their replace process
# the following statement sets a condition to wait for the _replace_zookeeper_property() function to complete
if len(hosts_replaced) > 0:
hosts_completed = my_queue_replace.get_hostnames(
len(hosts_s3) - len(hosts_replaced))
my_util.log.info(
"register_zookeeper: hosts that completed replacement: {}".
format(len(hosts_completed)))
# write a connection string to a temp file and upload to S3
my_file.write_connection_file(hosts_final)
# set state to 0 now that all hosts are configured
# state = 0, cluster is healthy
# state = 1, cluster requires replacement
my_file.write_zookeeper_state('0')
else:
# replace failed entries in zookeeper properties
for host in hosts_replaced:
my_file.replace_zookeeper_property(
host, hosts_new[hosts_replaced.index(host)])
my_queue_replace.send_hostname(my_hostname)
my_util.set_cmd_restart_zook(params['cmd_restart_zook'])
my_util.restart_zookeeper()
my_iid = my_util.exec_shell_command(
"curl -s http://169.254.169.254/latest/meta-data/instance-id")
my_zid = my_file.get_zookeeper_id()
my_ec2 = RegisterEC2(my_util, my_region)
result = my_ec2.add_tag(my_iid, 'KafkaId', my_zid)
if result == 200:
for tag in my_ec2.describe_tags(my_iid):
if tag['Key'] == 'KafkaId':
my_util.log.info("register_zookeeper: tag {}".format(tag))
import os
from register_ec2 import RegisterEC2
from register_file import RegisterFile
from register_util import RegisterUtil
my_basedir = os.path.dirname(os.path.realpath(__file__)).replace('/bin', '')
my_util = RegisterUtil("register_kafka", "{}/log".format(my_basedir))
my_hostname = my_util.exec_shell_command("curl -s http://169.254.169.254/latest/meta-data/local-ipv4")
my_region = my_util.exec_shell_command("curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | awk -F\\\" '{{print $4}}'")
my_iid = my_util.exec_shell_command("curl -s http://169.254.169.254/latest/meta-data/instance-id")
my_file = RegisterFile(my_util, my_region, my_basedir)
params = my_file.process_conf("{}/etc/register_kafka.conf".format(my_basedir))
my_file.set_config_brok(params['cfg_broker'])
my_bid = my_file.get_broker_id()
my_ec2 = RegisterEC2(my_util, my_region)
result = my_ec2.add_tag(my_iid, 'ApacheId', my_bid)
if result == 200:
for tag in my_ec2.describe_tags(my_iid):
if tag['Key'] == 'ApacheId':
my_util.log.info("register_kafka_tag: tag {}".format(tag))
def main():
my_basedir = os.path.dirname(os.path.realpath(__file__)).replace(
'/bin', '')
my_util = RegisterUtil("register_kafka", "{}/log".format(my_basedir))
my_util.log.info("register_kafka: beginning registration process")
my_util.log.info("register_kafka: using basedir={}".format(my_basedir))
# my_hostname = my_util.exec_shell_command("curl -s http://169.254.169.254/latest/meta-data/hostname")
my_hostname = my_util.exec_shell_command(
"curl -s http://169.254.169.254/latest/meta-data/local-ipv4")
my_region = my_util.exec_shell_command(
"curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | awk -F\\\" '{{print $4}}'"
)
my_util.log.info("register_zookeeper: hostname {}".format(my_hostname))
my_util.log.info("register_zookeeper: region {}".format(my_region))
my_asg = RegisterASG(my_util, my_region)
asg_name = my_asg.get_asg_name()
asg_desired_cap = my_asg.get_desired_capacity(asg_name)
asg_elb = my_asg.get_asg_elbs(asg_name)
my_util.log.info(
"register_kafka: instance is part of asg {}".format(asg_name))
my_util.log.info(
"register_kafka: asg desired capacity is currently {}".format(
asg_desired_cap))
my_util.log.info("register_kafka: asg is behind elb {}".format(asg_elb))
my_file = RegisterFile(my_util, my_region, my_basedir)
params = my_file.process_conf(
"{}/etc/register_kafka.conf".format(my_basedir))
my_file.set_bucket_name(params['s3_bucket'])
my_file.set_config_brok(params['cfg_broker'])
my_queue_brok = RegisterQueue(my_util, my_region, params['sqs_url_brok'])
my_util.set_cmd_restart_brok(params['cmd_restart_brok'])
cmd_brokids = '{} {}:2181 <<< "ls /brokers/ids" | grep "\["'.format(
params['zookshell'], params['zookeeper'])
# state = 0, brand new cluster build
# state = 1, existing cluster or failure replacement
state = my_file.get_broker_state()
firstrun = my_file.get_broker_firstrun()
my_util.log.info("register_kafka: firstrun: {}".format(firstrun))
my_util.log.info("register_kafka: state: {}".format(state))
if firstrun == '1' and state == '0':
# scenario: brand new instantiation
# need to send to broker queue
my_queue_brok.send_hostname(my_hostname)
# need to wait on broker queue
hosts_queue = my_queue_brok.get_hostnames(asg_desired_cap)
my_util.log.info("register_kafka: got {} hosts from queue".format(
len(hosts_queue)))
# write a ids string to a temp file and upload to S3
# note: setting first = 1001 is the default value for reserved.broker.id.max+1
first = int(params['broker_start'])
ids_init = range(first, first + int(asg_desired_cap))
my_file.write_ids_file(ids_init)
# write not first run locally
my_file.write_broker_firstrun('0')
# write not first run s3/globally
my_file.write_broker_state('1')
elif firstrun == '1' and state == '1':
# scenario: replacement
# need to first get active broker.ids
result = my_util.exec_shell_command(cmd_brokids).translate(
None, '[]').translate(None, ' ')
my_util.log.info("register_kafka: result: {}".format(result))
ids_active = sorted(result.split(","))
ids_active = [int(ident) for ident in ids_active]
my_util.log.info("register_kafka: active ids: {}".format(ids_active))
# need to discover bad.broker.id
# no longer programmatically generating ids_all but getting it from s3 now
# ids_all = range(ids_active[0], ids_active[0] + int(asg_desired_cap))
ids_all = my_file.get_ids_file()
ids_all = [int(ident) for ident in ids_all]
# need to handle failure + change in asg_desired_cap
if len(ids_all) != int(asg_desired_cap):
for add_id in range(1, int(asg_desired_cap) - len(ids_all)):
ids_all.append(ids_active[-1] + add_id)
my_util.log.info("register_kafka: all ids: {}".format(ids_all))
ids_missing = sorted(list(set(ids_all).difference(set(ids_active))))
my_util.log.info("register_kafka: missing ids: {}".format(ids_missing))
# setup puppet managed config file
my_file.init_broker_config()
# need to append the appropriate broker.id
# need to disable auto gen id
if len(ids_missing) > 0:
my_file.fix_broker_id(ids_missing[0])