sys.stderr.write( 'Starting {}, node id#{} of a {}-node ZooKeeper cluster...\n'.format( get_container_name(), ZOOKEEPER_NODE_ID, ZOOKEEPER_CLUSTER_SIZE)) else: sys.stderr.write( 'Starting {} as a single-node ZooKeeper cluster...\n'.format( get_container_name())) jvmflags = [ '-server', '-showversion', '-Dvisualvm.display.name="{}/{}"'.format(get_environment_name(), get_container_name()), ] jmx_port = get_port('jmx', -1) if jmx_port != -1: jvmflags += [ '-Djava.rmi.server.hostname={}'.format(get_container_host_address()), '-Dcom.sun.management.jmxremote.port={}'.format(jmx_port), '-Dcom.sun.management.jmxremote.rmi.port={}'.format(jmx_port), '-Dcom.sun.management.jmxremote.authenticate=false', '-Dcom.sun.management.jmxremote.local.only=false', '-Dcom.sun.management.jmxremote.ssl=false', ] os.environ['JVMFLAGS'] = ' '.join(jvmflags) + os.environ.get('JVM_OPTS', '') # Start ZooKeeper os.execl('bin/zkServer.sh', 'zookeeper', 'start-foreground')
def getKafkaZookeeperNode(name): return '{}:{}'.format( get_specific_host('zookeeper', name), get_specific_port('zookeeper', name, 'client')) def getKafkaZookeeper(): servers = _get_service_instance_names('zookeeper') print servers # servers = os.environ['ZOOKEEPER_SERVER_IDS'].split(',') return ','.join(map(getKafkaZookeeperNode, servers)) KAFKA_CONFIG_FILE = os.path.join('config', 'server.properties') KAFKA_LOGGING_CONFIG = os.path.join('config', 'log4j.properties') ZOOKEEPER_NODE_LIST = getKafkaZookeeper() sys.stderr.write(ZOOKEEPER_NODE_LIST) HOST_IP = get_container_host_address() LOG_PATTERN = "%d{yyyy'-'MM'-'dd'T'HH:mm:ss.SSSXXX} %-5p [%-35.35t] [%-36.36c]: %m%n" KAFKA_CONFIG_TEMPLATE = """# Kafka configuration for %(node_name)s broker.id=%(broker_id)d port=%(broker_port)d log.dirs=%(log_dirs)s num.partitions=%(num_partitions)d zookeeper.connect=%(zookeeper_connect)s zookeeper.connection.timeout.ms=6000 zookeeper.session.timeout.ms=6000 zookeeper.sync.time.ms=2000 advertised.host.name=%(host_ip)s
sys.stderr.write( 'Starting {}, node id#{} of a {}-node ZooKeeper cluster...\n' .format(get_container_name(), ZOOKEEPER_NODE_ID, ZOOKEEPER_CLUSTER_SIZE)) else: sys.stderr.write('Starting {} as a single-node ZooKeeper cluster...\n' .format(get_container_name())) jvmflags = [ '-server', '-showversion', '-Dvisualvm.display.name="{}/{}"'.format( get_environment_name(), get_container_name()), ] jmx_port = get_port('jmx', -1) if jmx_port != -1: jvmflags += [ '-Djava.rmi.server.hostname={}'.format(get_container_host_address()), '-Dcom.sun.management.jmxremote.port={}'.format(jmx_port), '-Dcom.sun.management.jmxremote.rmi.port={}'.format(jmx_port), '-Dcom.sun.management.jmxremote.authenticate=false', '-Dcom.sun.management.jmxremote.local.only=false', '-Dcom.sun.management.jmxremote.ssl=false', ] os.environ['JVMFLAGS'] = ' '.join(jvmflags) + ' ' + os.environ.get('JVM_OPTS', '') # Start ZooKeeper os.execl('bin/zkServer.sh', 'zookeeper', 'start-foreground')
print 'testing maestro and etcdmap' print orig.get_environment_name() print emap.get_environment_name() print 'print orig.get_service_name()' print orig.get_service_name() print emap.get_service_name() print 'get_container_name()' print orig.get_container_name() print emap.get_container_name() print 'get_container_host_address()' print orig.get_container_host_address() print emap.get_container_host_address() print 'get_container_internal_address()' print orig.get_container_internal_address() print emap.get_container_internal_address() print 'get_port(name, default = )' print orig.get_port('smtp') print emap.get_port('smtp') print 'get_node_list(service, ports=None)' print orig.get_node_list('cassandra') print emap.get_node_list('cassandra') print orig.get_node_list('cassandra', ports = ['rpc']) print emap.get_node_list('cassandra', ports = ['rpc'])
log4j.rootLogger=INFO,R log4j.appender.R=org.apache.log4j.RollingFileAppender log4j.appender.R.File=/var/log/%(service_name)s/%(container_name)s.log log4j.appender.R.MaxFileSize=100MB log4j.appender.R.MaxBackupIndex=10 log4j.appender.R.layout=org.apache.log4j.PatternLayout log4j.appender.R.layout.ConversionPattern=%(log_pattern)s """ replication = min(int(os.environ.get("REPLICATION", 2)), len(get_node_list("kafka"))) # Generate the Kafka configuration from the defined environment variables. config_model = { "node_name": get_container_name(), "broker_id": int(os.environ.get("BROKER_ID", 0)), "host_address": get_container_host_address(), "broker_port": get_port("broker", 9092), # Default log directory is /var/lib/kafka/logs. "log_dirs": os.environ.get("LOG_DIRS", "/var/lib/kafka/logs"), "num_partitions": int(os.environ.get("NUM_PARTITIONS", 8)), # Default retention is 7 days (168 hours). "retention_hours": int(os.environ.get("RETENTION_HOURS", 168)), # Default retention is only based on time. "retention_bytes": int(os.environ.get("RETENTION_BYTES", -1)), # Segment size (default is 0.5GB) "log_segment_bytes": int(os.environ.get("LOG_SEGMENT_BYTES", 536870912)), # Minimum interval between rolling new log segments (default 1 week) "log_roll_hours": int(os.environ.get("LOG_ROLL_HOURS", 24 * 7)), "zookeeper_nodes": ZOOKEEPER_NODE_LIST, "zookeeper_base": KAFKA_ZOOKEEPER_BASE, "flush_interval_ms": int(os.environ.get("FLUSH_INTERVAL_MS", 10000)),