def build_node_repr(name):
"""Build the representation of a node with peer and leader-election
ports."""
return '{}:{}:{}'.format(
get_specific_host(get_service_name(), name),
get_specific_port(get_service_name(), name, 'peer'),
get_specific_port(get_service_name(), name, 'leader_election'))
def build_node_repr(name):
"""Build the representation of a node with peer and leader-election
ports."""
return '{}:{}:{}'.format(
get_specific_host(get_service_name(), name),
get_specific_port(get_service_name(), name, 'peer'),
get_specific_port(get_service_name(), name, 'leader_election'))
def build_node_repr(name):
"""Build the representation of a node with peer and leader-election
ports."""
global conf
raw_peer_type_env = '%s_%s_peer_type' % (get_service_name(), name)
peer_type_env = raw_peer_type_env.replace('-', '_').upper()
peer_type = os.environ.get(peer_type_env, None)
if peer_type == 'observer':
conf['peerType'] = 'observer'
return '{}:{}:{}:observer'.format(
get_specific_host(get_service_name(), name),
get_specific_port(get_service_name(), name, 'peer'),
get_specific_port(get_service_name(), name, 'leader_election'))
else:
return '{}:{}:{}'.format(
get_specific_host(get_service_name(), name),
get_specific_port(get_service_name(), name, 'peer'),
get_specific_port(get_service_name(), name, 'leader_election'))
# Add the ZooKeeper node list with peer and leader election ports and figure
# out our own ID. ZOOKEEPER_SERVER_IDS contains a comma-separated list of
# node:id tuples describing the server ID of each node in the cluster, by its
# container name. If not specified, we assume single-node mode.
if os.environ.get('ZOOKEEPER_SERVER_IDS'):
servers = os.environ['ZOOKEEPER_SERVER_IDS'].split(',')
for server in servers:
node, id = server.split(':')
conf['server.{}'.format(id)] = build_node_repr(node)
if node == get_container_name():
ZOOKEEPER_NODE_ID = id
# Verify that the number of declared nodes matches the size of the cluster.
ZOOKEEPER_NODE_COUNT = len(get_node_list(get_service_name()))
ZOOKEEPER_CLUSTER_SIZE = len(
[i for i in conf.keys() if i.startswith('server.')])
# If no ZOOKEEPER_SERVER_IDS is defined, we expect to be in single-node mode so
# no more than one node can be declared in the cluster.
if ZOOKEEPER_CLUSTER_SIZE == 0 and ZOOKEEPER_NODE_COUNT != 1:
sys.stderr.write(
('Missing ZOOKEEPER_SERVER_IDS declaration for ' +
'{}-node ZooKeeper cluster!\n').format(ZOOKEEPER_NODE_COUNT))
sys.exit(1)
# If we got nodes from ZOOKEEPER_SERVER_IDS, we expect exactly the same number
# of nodes declared in the cluster.
if ZOOKEEPER_CLUSTER_SIZE > 0 and \
ZOOKEEPER_CLUSTER_SIZE != ZOOKEEPER_NODE_COUNT:
int(os.environ.get('REPLICA_SOCKET_TIMEOUT_MS', 2500)),
'replica_lag_max_ms':
int(os.environ.get('REPLICA_LAG_MAX_MS', 5000)),
'replica_lag_max_msgs':
int(os.environ.get('REPLICA_LAG_MAX_MSGS', 1000)),
'leader_rebalance':
str(os.environ.get('AUTO_LEADER_REBALANCE', 'false').lower() ==
'true').lower()
}
with open(KAFKA_CONFIG_FILE, 'w+') as conf:
conf.write(KAFKA_CONFIG_TEMPLATE % config_model)
# Setup the logging configuration.
logging_model = {
'service_name': get_service_name(),
'container_name': get_container_name(),
'log_pattern': LOG_PATTERN
}
with open(KAFKA_LOGGING_CONFIG, 'w+') as f:
f.write(KAFKA_LOGGING_TEMPLATE % logging_model)
# Ensure the existence of the ZooKeeper root node for Kafka
print 'Ensuring existance of the ZooKeeper zNode chroot path %s...' % KAFKA_ZOOKEEPER_BASE
def ensure_kafka_zk_path(retries=3):
while retries >= 0:
# Connect to the ZooKeeper nodes. Use a pretty large timeout in case they were
# just started. We should wait for them for a little while.
zk = KazooClient(hosts=ZOOKEEPER_NODE_LIST, timeout=30000)
def fix_collectd_file():
update_in_file('/etc/collectd/collectd.conf', '%%%HOSTNAME%%%',
get_service_name() + '.' + get_container_name())
import maestro.guestutils as orig
import maestro_etcd_map1 as emap
print 'testing maestro and etcdmap'
print orig.get_environment_name()
print emap.get_environment_name()
print 'print orig.get_service_name()'
print orig.get_service_name()
print emap.get_service_name()
print 'get_container_name()'
print orig.get_container_name()
print emap.get_container_name()
print 'get_container_host_address()'
print orig.get_container_host_address()
print emap.get_container_host_address()
print 'get_container_internal_address()'
print orig.get_container_internal_address()
print emap.get_container_internal_address()
print 'get_port(name, default = )'
print orig.get_port('smtp')
print emap.get_port('smtp')
print 'get_node_list(service, ports=None)'
print orig.get_node_list('cassandra')
print emap.get_node_list('cassandra')
# Add the ZooKeeper node list with peer and leader election ports and figure
# out our own ID. ZOOKEEPER_SERVER_IDS contains a comma-separated list of
# node:id tuples describing the server ID of each node in the cluster, by its
# container name. If not specified, we assume single-node mode.
if os.environ.get('ZOOKEEPER_SERVER_IDS'):
servers = os.environ['ZOOKEEPER_SERVER_IDS'].split(',')
for server in servers:
node, id = server.split(':')
conf['server.{}'.format(id)] = build_node_repr(node)
if node == get_container_name():
ZOOKEEPER_NODE_ID = id
# Verify that the number of declared nodes matches the size of the cluster.
ZOOKEEPER_NODE_COUNT = len(get_node_list(get_service_name()))
ZOOKEEPER_CLUSTER_SIZE = len(
[i for i in conf.keys() if i.startswith('server.')])
# If no ZOOKEEPER_SERVER_IDS is defined, we expect to be in single-node mode so
# no more than one node can be declared in the cluster.
if ZOOKEEPER_CLUSTER_SIZE == 0 and ZOOKEEPER_NODE_COUNT != 1:
sys.stderr.write(('Missing ZOOKEEPER_SERVER_IDS declaration for ' +
'{}-node ZooKeeper cluster!\n')
.format(ZOOKEEPER_NODE_COUNT))
sys.exit(1)
# If we got nodes from ZOOKEEPER_SERVER_IDS, we expect exactly the same number
# of nodes declared in the cluster.
if ZOOKEEPER_CLUSTER_SIZE > 0 and \
ZOOKEEPER_CLUSTER_SIZE != ZOOKEEPER_NODE_COUNT:
'num_threads': int(os.environ.get('NUM_THREADS', 8)),
'replication_factor': replication,
'num_replica_fetchers': int(os.environ.get('NUM_REPLICA_FETCHERS', 4)),
'replica_socket_timeout_ms': int(os.environ.get('REPLICA_SOCKET_TIMEOUT_MS', 2500)),
'replica_lag_max_ms': int(os.environ.get('REPLICA_LAG_MAX_MS', 5000)),
'replica_lag_max_msgs': int(os.environ.get('REPLICA_LAG_MAX_MSGS', 1000)),
'leader_rebalance': str(os.environ.get('AUTO_LEADER_REBALANCE', 'false').lower() == 'true').lower()
}
with open(KAFKA_CONFIG_FILE, 'w+') as conf:
conf.write(KAFKA_CONFIG_TEMPLATE % config_model)
# Setup the logging configuration.
logging_model = {
'service_name': get_service_name(),
'container_name': get_container_name(),
'log_pattern': LOG_PATTERN
}
with open(KAFKA_LOGGING_CONFIG, 'w+') as f:
f.write(KAFKA_LOGGING_TEMPLATE % logging_model)
# Ensure the existence of the ZooKeeper root node for Kafka
print 'Ensuring existance of the ZooKeeper zNode chroot path %s...' % KAFKA_ZOOKEEPER_BASE
def ensure_kafka_zk_path(retries=3):
while retries >= 0:
# Connect to the ZooKeeper nodes. Use a pretty large timeout in case they were
# just started. We should wait for them for a little while.
zk = KazooClient(hosts=ZOOKEEPER_NODE_LIST, timeout=30000)
"flush_interval_msgs": int(os.environ.get("FLUSH_INTERVAL_MSGS", 10000)),
"num_threads": int(os.environ.get("NUM_THREADS", 8)),
"replication_factor": replication,
"num_replica_fetchers": int(os.environ.get("NUM_REPLICA_FETCHERS", 4)),
"replica_socket_timeout_ms": int(os.environ.get("REPLICA_SOCKET_TIMEOUT_MS", 2500)),
"replica_lag_max_ms": int(os.environ.get("REPLICA_LAG_MAX_MS", 5000)),
"replica_lag_max_msgs": int(os.environ.get("REPLICA_LAG_MAX_MSGS", 1000)),
"leader_rebalance": str(os.environ.get("AUTO_LEADER_REBALANCE", "false").lower() == "true").lower(),
}
with open(KAFKA_CONFIG_FILE, "w+") as conf:
conf.write(KAFKA_CONFIG_TEMPLATE % config_model)
# Setup the logging configuration.
logging_model = {"service_name": get_service_name(), "container_name": get_container_name(), "log_pattern": LOG_PATTERN}
with open(KAFKA_LOGGING_CONFIG, "w+") as f:
f.write(KAFKA_LOGGING_TEMPLATE % logging_model)
# Ensure the existence of the ZooKeeper root node for Kafka
print "Ensuring existance of the ZooKeeper zNode chroot path %s..." % KAFKA_ZOOKEEPER_BASE
def ensure_kafka_zk_path(retries=3):
while retries >= 0:
# Connect to the ZooKeeper nodes. Use a pretty large timeout in case they were
# just started. We should wait for them for a little while.
zk = KazooClient(hosts=ZOOKEEPER_NODE_LIST, timeout=30000)
try:
zk.start()
zk.ensure_path(KAFKA_ZOOKEEPER_BASE)
ZOOKEEPER_NODE_ID = id
#os.mkdir(os.environ.get('ZOOKEEPER_DATADIR'))
myid_path = os.path.join(os.environ.get('ZOOKEEPER_DATADIR'), 'myid')
if os.path.isfile(myid_path):
os.remove(myid_path)
with open(myid_path, 'w+') as f:
f.write('%s\n' % ZOOKEEPER_NODE_ID)
with open (ZOOKEEPER_CONFIG_FILE, "r") as myfile:
data=myfile.read()
sys.stderr.write(data)
with open(ZOOKEEPER_LOG_CONFIG_FILE, 'w+') as f:
f.write("""# Log4j configuration, logs to rotating file
log4j.rootLogger=INFO,R
log4j.appender.R=org.apache.log4j.RollingFileAppender
log4j.appender.R.File=/var/log/%s/%s.log
log4j.appender.R.MaxFileSize=100MB
log4j.appender.R.MaxBackupIndex=10
log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.R.layout.ConversionPattern=%s
""" % (get_service_name(), get_container_name(),LOG_PATTERN))
#os.execl('bin/zkServer.sh', 'zookeeper', 'start-foreground')
os.execl('bin/zkServer.sh', 'zookeeper' ,'start-foreground')
sys.stderr.write('zookeeper over')
while True:
time.sleep(60)
sys.stderr.write('zookeeper over')
def fix_collectd_file():
update_in_file('/etc/collectd/collectd.conf', '%%%HOSTNAME%%%', get_service_name() + '.' + get_container_name())
