def run(config_file=None):
"""
Starts the binning process
Called here from main() when invoked from command line
but could equally import binning and call
binning.run(config_file)
"""
if config_file:
kafka_config.read_local_config(config_file)
with TopologyBuilder() as topology_builder:
topology_builder. \
source('prices', ['prices']). \
processor('binner', Binning, 'prices'). \
sink('result', 'bin-prices', 'binner')
wks = kafka_stream.KafkaStream(topology_builder, kafka_config)
wks.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
wks.close()
def run(config_file, binary_output):
kafka_config.read_local_config(config_file)
if binary_output:
kafka_config.VALUE_SERDE = 'examples.wordcount.custom_serde.StringIntSerde'
count_store = state_stores.create('counts'). \
with_string_keys(). \
with_integer_values(). \
in_memory(). \
build()
with TopologyBuilder() as topology_builder:
topology_builder. \
source('input-value', ['wks-wordcount-example-topic']). \
processor('count', WordCount, 'input-value'). \
state_store(count_store, 'count'). \
sink('output-count', 'wks-wordcount-example-count', 'count')
wks = kafka_streams.KafkaStreams(topology_builder, kafka_config)
wks.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
wks.close()
def _debug_run(config_file):
kafka_config.read_local_config(config_file)
double_store = state_stores.create('double_store'). \
with_integer_keys(). \
with_integer_values(). \
in_memory(). \
build()
with TopologyBuilder() as topology_builder:
topology_builder. \
source('input-value', ['wks-debug-example-topic-two']). \
processor('double', DoubleProcessor, 'input-value'). \
state_store(double_store, 'double'). \
sink('output-double', 'wks-debug-example-output', 'double')
wks = kafka_streams.KafkaStreams(topology_builder, kafka_config)
wks.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
wks.close()
def run(config_file):
kafka_config.read_local_config(config_file)
with TopologyBuilder() as topology_builder:
topology_builder. \
source('input-value', ['wks-wordcount-example-topic']). \
processor('count', WordCount, 'input-value'). \
state_store('counts', InMemoryKeyValueStore, 'count'). \
sink('output-count', 'wks-wordcount-example-count', 'count')
wks = kafka_streams.KafkaStreams(topology_builder, kafka_config)
wks.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
wks.close()
def _debug_run(config_file):
kafka_config.read_local_config(config_file)
with TopologyBuilder() as topology_builder:
topology_builder.source('input-value', ['example-topic']).processor(
'message', MessageProcessor,
'input-value').sink('output-value', 'example-topic-output',
'message')
wks = kafka_streams.KafkaStreams(topology_builder, kafka_config)
wks.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
wks.close()
def run(config_file):
kafka_config.read_local_config(config_file)
# Can also directly set config variables inline in Python
#kafka_config.KEY_SERDE = MySerde
with TopologyBuilder() as topology_builder:
topology_builder. \
source('inputs', ['prediction-inputs']). \
processor('count', WordCount, 'inputs'). \
sink('outputs', 'prediction-outputs', 'count')
wks = kafka_streams.KafkaStreams(topology_builder, kafka_config)
wks.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
wks.close()
def _debug_run(config_file):
kafka_config.read_local_config(config_file)
kafka_config.KEY_SERDE = serde_as_string(IntegerSerde)
kafka_config.VALUE_SERDE = serde_as_string(IntegerSerde)
with TopologyBuilder() as topology_builder:
topology_builder. \
source('input-value', ['wks-debug-example-topic-two']). \
processor('double', DoubleProcessor, 'input-value'). \
sink('output-double', 'wks-debug-example-output', 'double'). \
state_store('double-store', SimpleStore, 'double')
wks = kafka_streams.KafkaStreams(topology_builder, kafka_config)
wks.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
wks.close()
def run(config_file, binary_output):
kafka_config.read_local_config(config_file)
if binary_output:
kafka_config.VALUE_SERDE = 'examples.wordcount.custom_serde.StringIntSerde'
count_store = lambda name: ChangeLoggingKeyValueStore(name, InMemoryKeyValueStore)
with TopologyBuilder() as topology_builder:
topology_builder. \
source('input-value', ['wks-wordcount-example-topic']). \
processor('count', WordCount, 'input-value'). \
state_store('counts', count_store, 'count'). \
sink('output-count', 'wks-wordcount-example-count', 'count')
wks = kafka_streams.KafkaStreams(topology_builder, kafka_config)
wks.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
wks.close()
def _debug_run(config_file):
kafka_config.read_local_config(config_file)
# Can also directly set config variables inline in Python
#kafka_config.KEY_SERDE = MySerde
with TopologyBuilder() as topology_builder:
topology_builder. \
source('input-value', ['wks-debug-example-topic-two']). \
processor('double', DoubleProcessor, 'input-value'). \
sink('output-double', 'wks-debug-example-output', 'double'). \
state_store('double-store', SimpleStore, 'double')
wks = kafka_streams.KafkaStreams(topology_builder, kafka_config)
wks.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
wks.close()
