def application_setup(args):
# Parse user options for --depth and --internal-source
parser = argparse.ArgumentParser("Multi Partition Detector")
parser.add_argument("--depth", type=int, default=1,
help="The depth of the detector topology")
parser.add_argument("--gen-source", action='store_true',
help="Use an internal source for resilience tests")
parser.add_argument("--partitions", type=int, default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if pargs.gen_source:
print("Using internal source generator")
source = wallaroo.GenSourceConfig(MultiPartitionGenerator(pargs.partitions))
else:
print("Using TCP Source")
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)
p = wallaroo.source("Detector", source)
for x in range(pargs.depth):
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
p = p.to(trace_window)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args):
# Parse user options for --depth and --internal-source
parser = argparse.ArgumentParser("Multi Partition Detector")
parser.add_argument("--depth",
type=int,
default=1,
help="The depth of the detector topology")
parser.add_argument(
"--source",
choices=['tcp', 'gensource', 'alo'],
default='tcp',
help=("Choose source type for resilience tests. "
"'tcp' for standard TCP, 'gensource' for internal "
"generator source, and 'alo' for an external at-"
"least-once connector source."))
parser.add_argument(
"--partitions",
type=int,
default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if not '--cluster-initializer' in wallaroo._ARGS:
pargs.partitions = 0
source_name = "Detector"
if pargs.source == 'gensource':
print("Using internal source generator")
source = wallaroo.GenSourceConfig(
source_name, MultiPartitionGenerator(pargs.partitions))
elif pargs.source == 'tcp':
print("Using TCP Source")
in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_name, in_host, in_port, decoder)
elif pargs.source == 'alo':
print("Using at-least-once source")
in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.experimental.SourceConnectorConfig(
name=source_name,
encoder=encode_feed,
decoder=decode_feed,
host=in_host,
port=in_port,
cookie="cookie",
max_credits=67,
refill_credits=10)
p = wallaroo.source(source_name, source)
for x in range(pargs.depth):
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
p = p.to(trace_window)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args):
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
gen_source = wallaroo.GenSourceConfig(TransactionsGenerator())
transactions = wallaroo.source("Alerts (stateless)", gen_source)
pipeline = (transactions
.to(check_transaction)
.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode_alert)))
return wallaroo.build_application("Alerts (stateless)", pipeline)
def application_setup(args):
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
gen_source = wallaroo.GenSourceConfig(TransactionsGenerator())
transactions = wallaroo.source("Alerts (windowed)", gen_source)
pipeline = (transactions.key_by(extract_user).to(
wallaroo.range_windows(wallaroo.seconds(9)).with_slide(
wallaroo.seconds(3)).over(TotalAggregation)).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encode_alert)))
return wallaroo.build_application("Alerts (windowed)", pipeline)
def application_setup(args):
celsius_feed = wallaroo.GenSourceConfig('Gen celsius source',
CelsiusGenerator(999999))
fahrenheit_conversion = wallaroo.experimental.SinkConnectorConfig(
"fahrenheit_conversion",
encoder=encode_conversion,
decoder=decode_conversion,
port=7200,
cookie="Dragons Love Tacos!")
pipeline = (wallaroo.source(
"convert temperature readings",
celsius_feed).to(multiply).to(add).to_sink(fahrenheit_conversion))
return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args):
# Parse user options for --depth and --internal-source
parser = argparse.ArgumentParser("Multi Partition Detector")
parser.add_argument("--depth",
type=int,
default=1,
help="The depth of the detector topology")
parser.add_argument("--gen-source",
action='store_true',
help="Use an internal source for resilience tests")
parser.add_argument(
"--partitions",
type=int,
default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
partitions = []
ab = wallaroo.ApplicationBuilder("Multi Partition Detector")
if pargs.gen_source:
print("Using internal source generator")
source = wallaroo.GenSourceConfig(
MultiPartitionGenerator(pargs.partitions))
else:
print("Using TCP Source")
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)
ab.new_pipeline("Detector", source)
for x in range(pargs.depth):
ab.to(trace_id)
ab.to_state_partition(trace_window, WindowState, "state{}".format(x),
partition, partitions)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
parser = argparse.ArgumentParser("Window Detector")
parser.add_argument("--window-type",
default="tumbling",
choices=["tumbling", "sliding", "counting"])
parser.add_argument("--window-delay",
type=int,
default=0,
help=("Window delay"
"size in milliseconds. (Default: 0)"))
parser.add_argument("--window-size",
type=int,
default=50,
help=("Window size in"
"milliseconds or units. (Default: 50)"))
parser.add_argument("--window-slide",
type=int,
default=25,
help=("Window slide size, in milliseconds. "
"(Default: 25)"))
parser.add_argument(
"--window-late-policy",
default="drop",
choices=["drop", "fire-per-message", "place-in-oldest-window"])
parser.add_argument(
"--source",
choices=['tcp', 'gensource', 'alo'],
default='tcp',
help=("Choose source type for resilience tests. "
"'tcp' for standard TCP, 'gensource' for internal "
"generator source, and 'alo' for an external at-"
"least-once connector source."))
parser.add_argument(
"--partitions",
type=int,
default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if not '--cluster-initializer' in wallaroo._ARGS:
pargs.partitions = 0
source_name = "Detector"
if pargs.source == 'gensource':
print("Using internal source generator")
source = wallaroo.GenSourceConfig(
source_name, MultiPartitionGenerator(pargs.partitions))
elif pargs.source == 'tcp':
print("Using TCP Source")
_, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(source_name, in_host, in_port,
decoder)
elif pargs.source == 'alo':
print("Using at-least-once source")
_, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.experimental.SourceConnectorConfig(
name=source_name,
encoder=encode_feed,
decoder=decode_feed,
host=in_host,
port=in_port,
cookie="cookie",
max_credits=67,
refill_credits=10)
p = wallaroo.source(source_name, source)
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
# Programmatically construct the window type and arguments
if pargs.window_type == 'counting':
print("Using window size: {} units".format(pargs.window_size))
window = wallaroo.count_windows(pargs.window_size)
else:
print("Using window size: {} ms".format(pargs.window_size))
window = wallaroo.range_windows(
wallaroo.milliseconds(pargs.window_size))
if pargs.window_delay:
print("Using window_delay: {} ms".format(pargs.window_delay))
window = window.with_delay(
wallaroo.milliseconds(pargs.window_delay))
if pargs.window_type == 'sliding':
print("Using window_slide: {} ms".format(pargs.window_slide))
window = window.with_slide(
wallaroo.milliseconds(pargs.window_slide))
if pargs.window_late_policy:
## NOTE: The Wallaroo default policy for late window data is "drop".
## However, it's quite likely that that policy can cause
## intermittent test failures.
if pargs.window_late_policy == "drop":
print(
"\n\nWARNING: drop policy can cause intermittent test failures!\n\n"
)
print("Using window_late_policy: {}".format(
pargs.window_late_policy))
window = window.with_late_data_policy(pargs.window_late_policy)
# add the window to the topology
p = p.to(window.over(Collect))
p = p.to(split_accumulated)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Tumbling Time Window Detector", p)
def application_setup(args):
parser = argparse.ArgumentParser("Window Detector")
parser.add_argument("--window-type",
default="tumbling",
choices=["tumbling", "sliding", "counting"])
parser.add_argument("--window-delay",
type=int,
default=0,
help=("Window delay"
"size in milliseconds. (Default: 0)"))
parser.add_argument("--window-size",
type=int,
default=50,
help=("Window size in"
"milliseconds or units. (Default: 50)"))
parser.add_argument("--window-slide",
type=int,
default=25,
help=("Window slide size, in milliseconds. "
"(Default: 25)"))
parser.add_argument("--gen-source",
action='store_true',
help="Use an internal source for resilience tests")
parser.add_argument(
"--partitions",
type=int,
default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if pargs.gen_source:
print("Using internal source generator")
source = wallaroo.GenSourceConfig(
MultiPartitionGenerator(pargs.partitions))
else:
print("Using TCP Source")
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)
p = wallaroo.source("{} window".format(pargs.window_type), source)
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
# Programmatically construct the window type and arguments
if pargs.window_type == 'counting':
print("Using window size: {} units".format(pargs.window_size))
window = wallaroo.count_windows(pargs.window_size)
else:
print("Using window size: {} ms".format(pargs.window_size))
window = wallaroo.range_windows(
wallaroo.milliseconds(pargs.window_size))
if pargs.window_delay:
print("Using window_delay: {} ms".format(pargs.window_delay))
window = window.with_delay(
wallaroo.milliseconds(pargs.window_delay))
if pargs.window_type == 'sliding':
print("Using window_slide: {} ms".format(pargs.window_slide))
window = window.with_slide(
wallaroo.milliseconds(pargs.window_slide))
# add the window to the topology
p = p.to(window.over(Collect))
p = p.to(split_accumulated)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Tumbling Time Window Detector", p)
