def application_setup(args):
# Parse user options for --depth and --internal-source
parser = argparse.ArgumentParser("Multi Partition Detector")
parser.add_argument("--depth", type=int, default=1,
help="The depth of the detector topology")
parser.add_argument("--gen-source", action='store_true',
help="Use an internal source for resilience tests")
parser.add_argument("--partitions", type=int, default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if pargs.gen_source:
print("Using internal source generator")
source = wallaroo.GenSourceConfig(MultiPartitionGenerator(pargs.partitions))
else:
print("Using TCP Source")
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)
p = wallaroo.source("Detector", source)
for x in range(pargs.depth):
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
p = p.to(trace_window)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args):
values = wallaroo.source("convert",
wallaroo.DefaultKafkaSourceCLIParser(decoder))
pipeline = (values.to(multiply).to(add).to_sink(
wallaroo.DefaultKafkaSinkCLIParser(encoder)))
return wallaroo.build_application("Celsius to Fahrenheit with Kafka",
pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
pipeline = (wallaroo.source(
"Dummy", wallaroo.TCPSourceConfig(in_host, in_port, decoder)).to(
count).key_by(partition).to(count_partitioned).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("Dummy", pipeline)
def application_setup(args):
# Parse user options for --depth and --internal-source
parser = argparse.ArgumentParser("Multi Partition Detector")
parser.add_argument("--depth",
type=int,
default=1,
help="The depth of the detector topology")
parser.add_argument(
"--source",
choices=['tcp', 'gensource', 'alo'],
default='tcp',
help=("Choose source type for resilience tests. "
"'tcp' for standard TCP, 'gensource' for internal "
"generator source, and 'alo' for an external at-"
"least-once connector source."))
parser.add_argument(
"--partitions",
type=int,
default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if not '--cluster-initializer' in wallaroo._ARGS:
pargs.partitions = 0
source_name = "Detector"
if pargs.source == 'gensource':
print("Using internal source generator")
source = wallaroo.GenSourceConfig(
source_name, MultiPartitionGenerator(pargs.partitions))
elif pargs.source == 'tcp':
print("Using TCP Source")
in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_name, in_host, in_port, decoder)
elif pargs.source == 'alo':
print("Using at-least-once source")
in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.experimental.SourceConnectorConfig(
name=source_name,
encoder=encode_feed,
decoder=decode_feed,
host=in_host,
port=in_port,
cookie="cookie",
max_credits=67,
refill_credits=10)
p = wallaroo.source(source_name, source)
for x in range(pargs.depth):
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
p = p.to(trace_window)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
lines = wallaroo.source(
"Split and Count", wallaroo.TCPSourceConfig(in_host, in_port, decoder))
pipeline = (lines.to(split).key_by(extract_word).to(count_word).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("Word Count Application", pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
votes = wallaroo.source(
"alphabet", wallaroo.TCPSourceConfig(in_host, in_port, decoder))
pipeline = (votes.key_by(extract_letter).to(add_votes).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("alphabet", pipeline)
def application_setup(args):
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
gen_source = wallaroo.GenSourceConfig(TransactionsGenerator())
transactions = wallaroo.source("Alerts (stateless)", gen_source)
pipeline = (transactions
.to(check_transaction)
.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode_alert)))
return wallaroo.build_application("Alerts (stateless)", pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
inputs = wallaroo.source(
"Sequence Window", wallaroo.TCPSourceConfig(in_host, in_port, decoder))
pipeline = (inputs.key_by(constant).to(maybe_one_to_many).key_by(
extract_window).to(observe_new_value).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("Sequence Window", pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
inputs = wallaroo.source(
"Celsius Conversion",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
pipeline = (inputs.to(multiply).to(add).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args):
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
gen_source = wallaroo.GenSourceConfig(TransactionsGenerator())
transactions = wallaroo.source("Alerts (windowed)", gen_source)
pipeline = (transactions.key_by(extract_user).to(
wallaroo.range_windows(wallaroo.seconds(9)).with_slide(
wallaroo.seconds(3)).over(TotalAggregation)).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encode_alert)))
return wallaroo.build_application("Alerts (windowed)", pipeline)
def application_setup(args):
celsius_feed = wallaroo.experimental.SourceConnectorConfig(
"celsius_feed", encoder=encode_feed, decoder=decode_feed, port=7100)
fahrenheit_conversion = wallaroo.experimental.SinkConnectorConfig(
"fahrenheit_conversion",
encoder=encode_conversion,
decoder=decode_conversion,
port=7200)
pipeline = (wallaroo.source(
"convert temperature readings",
celsius_feed).to(multiply).to(add).to_sink(fahrenheit_conversion))
return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args):
global PID
PID = str(os.getpid())
in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
inputs = wallaroo.source(
"App", wallaroo.TCPSourceConfig(in_name, in_host, in_port, decode))
pipeline = (inputs.key_by(extract_key).to(classify).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encode)))
return wallaroo.build_application("Parallel App", pipeline)
def application_setup(args):
celsius_feed = wallaroo.GenSourceConfig('Gen celsius source',
CelsiusGenerator(999999))
fahrenheit_conversion = wallaroo.experimental.SinkConnectorConfig(
"fahrenheit_conversion",
encoder=encode_conversion,
decoder=decode_conversion,
port=7200,
cookie="Dragons Love Tacos!")
pipeline = (wallaroo.source(
"convert temperature readings",
celsius_feed).to(multiply).to(add).to_sink(fahrenheit_conversion))
return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args):
parse_delay(args)
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
nonce_source = wallaroo.TCPSourceConfig(in_host, in_port, nonce_decoder)
ok_sink = wallaroo.TCPSinkConfig(out_host, out_port, ok_encoder)
inputs = wallaroo.source("Counting Sheep", nonce_source)
pipeline = (inputs.
.to(process_nonce)
.key_by(extract_key)
.to(busy_sleep)
.to_sink(ok_sink))
return wallaroo.build_application("Counting Sheep", pipeline)
def application_setup(args):
input_addrs = wallaroo.tcp_parse_input_addrs(args)
order_host, order_port = input_addrs[0]
nbbo_host, nbbo_port = input_addrs[1]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
orders = wallaroo.source(
"Orders",
wallaroo.TCPSourceConfig(order_host, order_port, order_decoder))
market_data = wallaroo.source(
"Market Data",
wallaroo.TCPSourceConfig(nbbo_host, nbbo_port, market_data_decoder))
pipeline = (orders.merge(market_data).key_by(extract_symbol).to(
check_market_data).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, order_result_encoder)))
return wallaroo.build_application("Market Spread", pipeline)
def application_setup(args):
input_addrs = wallaroo.tcp_parse_input_addrs(args)
inputs = {ip[0]: {"host": ip[1], "port": ip[2]} for ip in input_addrs}
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
orders = wallaroo.source(
"Orders",
wallaroo.TCPSourceConfig("Orders", inputs["Orders"]["host"],
inputs["Orders"]["port"], decode_order))
market_data = wallaroo.source(
"Market Data",
wallaroo.TCPSourceConfig("Market Data", inputs["Market Data"]["host"],
inputs["Market Data"]["port"],
decode_market_data))
pipeline = (orders.merge(market_data).key_by(extract_symbol).to(
check_market_data).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encode_order_result)))
return wallaroo.build_application("Market Spread", pipeline)
def application_setup(args):
# Parse user options
parser = argparse.ArgumentParser("Topology Test Generator")
parser_add_args(parser)
pargs, _ = parser.parse_known_args(args)
app_name = "topology test"
pipe_name = "topology test pipeline"
print("Using TCP Source")
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)
inputs = wallaroo.source(pipe_name, source)
# programmatically add computations
topology = Topology(pargs.topology)
pipeline = topology.build(inputs)
print("Using TCP Sink")
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
pipeline = pipeline.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application(app_name, pipeline)
def application_setup(args):
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
#source_config = wallaroo.TCPSourceConfig("celsius", in_host, in_port, decoder)
source_config = wallaroo.experimental.SourceConnectorConfig(
"celsius_feed",
encoder=encode_feed,
decoder=decode_feed,
host=in_host,
port=in_port,
cookie="Dragons Love Tacos!",
max_credits=10,
refill_credits=8)
#sink_config = wallaroo.experimental.SinkConnectorConfig(
# "fahrenheit_conversion",
# encoder=encode_conversion,
# decoder=decode_conversion,
# port=7200)
sink_config = wallaroo.TCPSinkConfig(out_host, out_port, encode_conversion)
pipeline = (wallaroo.source(
"convert temperature readings",
source_config).to(multiply).to(add).to_sink(sink_config))
return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args):
# ab.source_connector("celsius_feed",
# encoder=encode_feed,
# decoder=decode_feed)
# ab.sink_connector("fahrenheit_conversion",
# encoder=encode_conversion,
# decoder=decode_conversion)
# ab.new_pipeline("Celsius Conversion", "celsius_feed")
# ab.to(multiply)
# ab.to(add)
# ab.to_sink("fahrenheit_conversion")
# return ab.build()
#!@
values = wallaroo.source("convert",
wallaroo.???(decoder))
pipeline = (values
.to(multiply)
.to(add)
.to_sink(wallaroo.DefaultKafkaSinkCLIParser(encoder)))
return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args):
parser = argparse.ArgumentParser("Window Detector")
parser.add_argument("--window-type",
default="tumbling",
choices=["tumbling", "sliding", "counting"])
parser.add_argument("--window-delay",
type=int,
default=0,
help=("Window delay"
"size in milliseconds. (Default: 0)"))
parser.add_argument("--window-size",
type=int,
default=50,
help=("Window size in"
"milliseconds or units. (Default: 50)"))
parser.add_argument("--window-slide",
type=int,
default=25,
help=("Window slide size, in milliseconds. "
"(Default: 25)"))
parser.add_argument("--gen-source",
action='store_true',
help="Use an internal source for resilience tests")
parser.add_argument(
"--partitions",
type=int,
default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if pargs.gen_source:
print("Using internal source generator")
source = wallaroo.GenSourceConfig(
MultiPartitionGenerator(pargs.partitions))
else:
print("Using TCP Source")
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)
p = wallaroo.source("{} window".format(pargs.window_type), source)
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
# Programmatically construct the window type and arguments
if pargs.window_type == 'counting':
print("Using window size: {} units".format(pargs.window_size))
window = wallaroo.count_windows(pargs.window_size)
else:
print("Using window size: {} ms".format(pargs.window_size))
window = wallaroo.range_windows(
wallaroo.milliseconds(pargs.window_size))
if pargs.window_delay:
print("Using window_delay: {} ms".format(pargs.window_delay))
window = window.with_delay(
wallaroo.milliseconds(pargs.window_delay))
if pargs.window_type == 'sliding':
print("Using window_slide: {} ms".format(pargs.window_slide))
window = window.with_slide(
wallaroo.milliseconds(pargs.window_slide))
# add the window to the topology
p = p.to(window.over(Collect))
p = p.to(split_accumulated)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Tumbling Time Window Detector", p)
def application_setup(args):
parser = argparse.ArgumentParser("Window Detector")
parser.add_argument("--window-type",
default="tumbling",
choices=["tumbling", "sliding", "counting"])
parser.add_argument("--window-delay",
type=int,
default=0,
help=("Window delay"
"size in milliseconds. (Default: 0)"))
parser.add_argument("--window-size",
type=int,
default=50,
help=("Window size in"
"milliseconds or units. (Default: 50)"))
parser.add_argument("--window-slide",
type=int,
default=25,
help=("Window slide size, in milliseconds. "
"(Default: 25)"))
parser.add_argument(
"--window-late-policy",
default="drop",
choices=["drop", "fire-per-message", "place-in-oldest-window"])
parser.add_argument(
"--source",
choices=['tcp', 'gensource', 'alo'],
default='tcp',
help=("Choose source type for resilience tests. "
"'tcp' for standard TCP, 'gensource' for internal "
"generator source, and 'alo' for an external at-"
"least-once connector source."))
parser.add_argument(
"--partitions",
type=int,
default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if not '--cluster-initializer' in wallaroo._ARGS:
pargs.partitions = 0
source_name = "Detector"
if pargs.source == 'gensource':
print("Using internal source generator")
source = wallaroo.GenSourceConfig(
source_name, MultiPartitionGenerator(pargs.partitions))
elif pargs.source == 'tcp':
print("Using TCP Source")
_, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(source_name, in_host, in_port,
decoder)
elif pargs.source == 'alo':
print("Using at-least-once source")
_, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.experimental.SourceConnectorConfig(
name=source_name,
encoder=encode_feed,
decoder=decode_feed,
host=in_host,
port=in_port,
cookie="cookie",
max_credits=67,
refill_credits=10)
p = wallaroo.source(source_name, source)
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
# Programmatically construct the window type and arguments
if pargs.window_type == 'counting':
print("Using window size: {} units".format(pargs.window_size))
window = wallaroo.count_windows(pargs.window_size)
else:
print("Using window size: {} ms".format(pargs.window_size))
window = wallaroo.range_windows(
wallaroo.milliseconds(pargs.window_size))
if pargs.window_delay:
print("Using window_delay: {} ms".format(pargs.window_delay))
window = window.with_delay(
wallaroo.milliseconds(pargs.window_delay))
if pargs.window_type == 'sliding':
print("Using window_slide: {} ms".format(pargs.window_slide))
window = window.with_slide(
wallaroo.milliseconds(pargs.window_slide))
if pargs.window_late_policy:
## NOTE: The Wallaroo default policy for late window data is "drop".
## However, it's quite likely that that policy can cause
## intermittent test failures.
if pargs.window_late_policy == "drop":
print(
"\n\nWARNING: drop policy can cause intermittent test failures!\n\n"
)
print("Using window_late_policy: {}".format(
pargs.window_late_policy))
window = window.with_late_data_policy(pargs.window_late_policy)
# add the window to the topology
p = p.to(window.over(Collect))
p = p.to(split_accumulated)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Tumbling Time Window Detector", p)
