sys.stderr.write(
'Starting {}, node id#{} of a {}-node ZooKeeper cluster...\n'.format(
get_container_name(), ZOOKEEPER_NODE_ID, ZOOKEEPER_CLUSTER_SIZE))
else:
sys.stderr.write(
'Starting {} as a single-node ZooKeeper cluster...\n'.format(
get_container_name()))
jvmflags = [
'-server',
'-showversion',
'-Dvisualvm.display.name="{}/{}"'.format(get_environment_name(),
get_container_name()),
]
jmx_port = get_port('jmx', -1)
if jmx_port != -1:
jvmflags += [
'-Djava.rmi.server.hostname={}'.format(get_container_host_address()),
'-Dcom.sun.management.jmxremote.port={}'.format(jmx_port),
'-Dcom.sun.management.jmxremote.rmi.port={}'.format(jmx_port),
'-Dcom.sun.management.jmxremote.authenticate=false',
'-Dcom.sun.management.jmxremote.local.only=false',
'-Dcom.sun.management.jmxremote.ssl=false',
]
os.environ['JVMFLAGS'] = ' '.join(jvmflags) + os.environ.get('JVM_OPTS', '')
# Start ZooKeeper
os.execl('bin/zkServer.sh', 'zookeeper', 'start-foreground')
def getKafkaZookeeperNode(name):
return '{}:{}'.format(
get_specific_host('zookeeper', name),
get_specific_port('zookeeper', name, 'client'))
def getKafkaZookeeper():
servers = _get_service_instance_names('zookeeper')
print servers
# servers = os.environ['ZOOKEEPER_SERVER_IDS'].split(',')
return ','.join(map(getKafkaZookeeperNode, servers))
KAFKA_CONFIG_FILE = os.path.join('config', 'server.properties')
KAFKA_LOGGING_CONFIG = os.path.join('config', 'log4j.properties')
ZOOKEEPER_NODE_LIST = getKafkaZookeeper()
sys.stderr.write(ZOOKEEPER_NODE_LIST)
HOST_IP = get_container_host_address()
LOG_PATTERN = "%d{yyyy'-'MM'-'dd'T'HH:mm:ss.SSSXXX} %-5p [%-35.35t] [%-36.36c]: %m%n"
KAFKA_CONFIG_TEMPLATE = """# Kafka configuration for %(node_name)s
broker.id=%(broker_id)d
port=%(broker_port)d
log.dirs=%(log_dirs)s
num.partitions=%(num_partitions)d
zookeeper.connect=%(zookeeper_connect)s
zookeeper.connection.timeout.ms=6000
zookeeper.session.timeout.ms=6000
zookeeper.sync.time.ms=2000
advertised.host.name=%(host_ip)s
sys.stderr.write(
'Starting {}, node id#{} of a {}-node ZooKeeper cluster...\n'
.format(get_container_name(), ZOOKEEPER_NODE_ID,
ZOOKEEPER_CLUSTER_SIZE))
else:
sys.stderr.write('Starting {} as a single-node ZooKeeper cluster...\n'
.format(get_container_name()))
jvmflags = [
'-server',
'-showversion',
'-Dvisualvm.display.name="{}/{}"'.format(
get_environment_name(), get_container_name()),
]
jmx_port = get_port('jmx', -1)
if jmx_port != -1:
jvmflags += [
'-Djava.rmi.server.hostname={}'.format(get_container_host_address()),
'-Dcom.sun.management.jmxremote.port={}'.format(jmx_port),
'-Dcom.sun.management.jmxremote.rmi.port={}'.format(jmx_port),
'-Dcom.sun.management.jmxremote.authenticate=false',
'-Dcom.sun.management.jmxremote.local.only=false',
'-Dcom.sun.management.jmxremote.ssl=false',
]
os.environ['JVMFLAGS'] = ' '.join(jvmflags) + ' ' + os.environ.get('JVM_OPTS', '')
# Start ZooKeeper
os.execl('bin/zkServer.sh', 'zookeeper', 'start-foreground')
print 'testing maestro and etcdmap'
print orig.get_environment_name()
print emap.get_environment_name()
print 'print orig.get_service_name()'
print orig.get_service_name()
print emap.get_service_name()
print 'get_container_name()'
print orig.get_container_name()
print emap.get_container_name()
print 'get_container_host_address()'
print orig.get_container_host_address()
print emap.get_container_host_address()
print 'get_container_internal_address()'
print orig.get_container_internal_address()
print emap.get_container_internal_address()
print 'get_port(name, default = )'
print orig.get_port('smtp')
print emap.get_port('smtp')
print 'get_node_list(service, ports=None)'
print orig.get_node_list('cassandra')
print emap.get_node_list('cassandra')
print orig.get_node_list('cassandra', ports = ['rpc'])
print emap.get_node_list('cassandra', ports = ['rpc'])
log4j.rootLogger=INFO,R
log4j.appender.R=org.apache.log4j.RollingFileAppender
log4j.appender.R.File=/var/log/%(service_name)s/%(container_name)s.log
log4j.appender.R.MaxFileSize=100MB
log4j.appender.R.MaxBackupIndex=10
log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.R.layout.ConversionPattern=%(log_pattern)s
"""
replication = min(int(os.environ.get("REPLICATION", 2)), len(get_node_list("kafka")))
# Generate the Kafka configuration from the defined environment variables.
config_model = {
"node_name": get_container_name(),
"broker_id": int(os.environ.get("BROKER_ID", 0)),
"host_address": get_container_host_address(),
"broker_port": get_port("broker", 9092),
# Default log directory is /var/lib/kafka/logs.
"log_dirs": os.environ.get("LOG_DIRS", "/var/lib/kafka/logs"),
"num_partitions": int(os.environ.get("NUM_PARTITIONS", 8)),
# Default retention is 7 days (168 hours).
"retention_hours": int(os.environ.get("RETENTION_HOURS", 168)),
# Default retention is only based on time.
"retention_bytes": int(os.environ.get("RETENTION_BYTES", -1)),
# Segment size (default is 0.5GB)
"log_segment_bytes": int(os.environ.get("LOG_SEGMENT_BYTES", 536870912)),
# Minimum interval between rolling new log segments (default 1 week)
"log_roll_hours": int(os.environ.get("LOG_ROLL_HOURS", 24 * 7)),
"zookeeper_nodes": ZOOKEEPER_NODE_LIST,
"zookeeper_base": KAFKA_ZOOKEEPER_BASE,
"flush_interval_ms": int(os.environ.get("FLUSH_INTERVAL_MS", 10000)),
