def application_setup(args):
input_addrs = wallaroo.tcp_parse_input_addrs(args)
order_host, order_port = input_addrs[0]
nbbo_host, nbbo_port = input_addrs[1]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
symbol_partitions = [
str_to_partition(x.rjust(4)) for x in load_valid_symbols()
]
ab = wallaroo.ApplicationBuilder("market-spread")
ab.new_pipeline(
"Orders",
wallaroo.TCPSourceConfig(
order_host, order_port, OrderDecoder())).to_state_partition_u64(
CheckOrder(), SymbolDataBuilder(), "symbol-data",
SymbolPartitionFunction(), symbol_partitions).to_sink(
wallaroo.TCPSinkConfig(
out_host, out_port,
OrderResultEncoder())).new_pipeline(
"Market Data",
wallaroo.TCPSourceConfig(
nbbo_host, nbbo_port,
MarketDataDecoder())).to_state_partition_u64(
UpdateMarketData(), SymbolDataBuilder(),
"symbol-data", SymbolPartitionFunction(),
symbol_partitions).done()
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ll_host, ll_port = wallaroo.tcp_parse_input_addrs(args)[1]
ab = wallaroo.ApplicationBuilder("gdax alerts")
# Pipeline to store alerts and the corresponding user
ab.new_pipeline("Load users created alerts",
wallaroo.TCPSourceConfig(ll_host, ll_port, decoder))
ab.to_stateful(save_alert, Alerts, "user set alerts")
ab.done()
# Pipeline to calculate current bitcoin price from gdax
ab.new_pipeline("Average BTC price",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to_parallel(extract_price)
# State object, current_price
# Update the moving
ab.to_stateful(calculate_and_update_average_price, BTCPrice,
"Calculate and update the average price")
# Alerts are map of `price: [user_ids]`
# Alert.update lookup users, return and replace with None
# Send to another application, that then calls a cellery task
ab.to_stateful(maybe_send_alerts_based_on_average_price, Alerts,
"user set alerts")
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
# Parse user options for --depth and --internal-source
parser = argparse.ArgumentParser("Multi Partition Detector")
parser.add_argument("--depth", type=int, default=1,
help="The depth of the detector topology")
parser.add_argument("--gen-source", action='store_true',
help="Use an internal source for resilience tests")
parser.add_argument("--partitions", type=int, default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if pargs.gen_source:
print("Using internal source generator")
source = wallaroo.GenSourceConfig(MultiPartitionGenerator(pargs.partitions))
else:
print("Using TCP Source")
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)
p = wallaroo.source("Detector", source)
for x in range(pargs.depth):
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
p = p.to(trace_window)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args):
input_addrs = wallaroo.tcp_parse_input_addrs(args)
ll_host, ll_port = input_addrs[0]
cc_host, cc_port = input_addrs[1]
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
initial_partitions = range(0, 10)
ab = wallaroo.ApplicationBuilder("Ad Tech Application")
ab.new_pipeline("Load loyalty customers",
wallaroo.TCPSourceConfig(ll_host, ll_port, ll_decoder))
ab.to_state_partition(save_customer, LoyaltyCustomers, "loyalty customers",
extract_conversion_key, initial_partitions)
ab.done()
ab.new_pipeline("Conversions",
wallaroo.TCPSourceConfig(cc_host, cc_port, cc_decoder))
ab.to_state_partition(process_email_add_customer, LoyaltyCustomers,
"loyalty customers", extract_conversion_key,
initial_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, cc_encoder))
return ab.build()
def application_setup(args):
input_addrs = wallaroo.tcp_parse_input_addrs(args)
order_host, order_port = input_addrs[0]
nbbo_host, nbbo_port = input_addrs[1]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
symbol_partitions = [str_to_partition(x.rjust(4)) for x in
load_valid_symbols()]
ab = wallaroo.ApplicationBuilder("market-spread")
ab.new_pipeline(
"Orders",
wallaroo.TCPSourceConfig(order_host, order_port, order_decoder)
).to_state_partition_u64(
check_order, SymbolData, "symbol-data",
symbol_partition_function, symbol_partitions
).to_sink(wallaroo.TCPSinkConfig(out_host, out_port,
order_result_encoder)
).new_pipeline(
"Market Data",
wallaroo.TCPSourceConfig(nbbo_host, nbbo_port,
market_data_decoder)
).to_state_partition_u64(
update_market_data, SymbolData, "symbol-data",
symbol_partition_function, symbol_partitions
).done()
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
pipeline = (wallaroo.source(
"Dummy", wallaroo.TCPSourceConfig(in_host, in_port, decoder)).to(
count).key_by(partition).to(count_partitioned).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("Dummy", pipeline)
def application_setup(args):
# Parse user options for --depth and --internal-source
parser = argparse.ArgumentParser("Multi Partition Detector")
parser.add_argument("--depth",
type=int,
default=1,
help="The depth of the detector topology")
parser.add_argument(
"--source",
choices=['tcp', 'gensource', 'alo'],
default='tcp',
help=("Choose source type for resilience tests. "
"'tcp' for standard TCP, 'gensource' for internal "
"generator source, and 'alo' for an external at-"
"least-once connector source."))
parser.add_argument(
"--partitions",
type=int,
default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
if not '--cluster-initializer' in wallaroo._ARGS:
pargs.partitions = 0
source_name = "Detector"
if pargs.source == 'gensource':
print("Using internal source generator")
source = wallaroo.GenSourceConfig(
source_name, MultiPartitionGenerator(pargs.partitions))
elif pargs.source == 'tcp':
print("Using TCP Source")
in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_name, in_host, in_port, decoder)
elif pargs.source == 'alo':
print("Using at-least-once source")
in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.experimental.SourceConnectorConfig(
name=source_name,
encoder=encode_feed,
decoder=decode_feed,
host=in_host,
port=in_port,
cookie="cookie",
max_credits=67,
refill_credits=10)
p = wallaroo.source(source_name, source)
for x in range(pargs.depth):
p = p.key_by(extract_key)
p = p.to(trace_id)
p = p.key_by(extract_key)
p = p.to(trace_window)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Trending Hashtags")
ab.new_pipeline("Tweets_new",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to(HashtagFinder)
ab.to_stateful(ComputeHashtags(), HashtagsStateBuilder(), "hashtags state")
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Add2")
ab.new_pipeline("add_2",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to(add_2)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("alphabet")
ab.new_pipeline("alphabet",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to_stateful(add_votes, AllVotes, "letter state")
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Apache Log file analysis")
ab.new_pipeline("convert",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to_stateful(Count(), CounterBuilder(), "status counter")
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
lines = wallaroo.source(
"Split and Count", wallaroo.TCPSourceConfig(in_host, in_port, decoder))
pipeline = (lines.to(split).key_by(extract_word).to(count_word).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("Word Count Application", pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Reverse Word")
ab.new_pipeline("reverse",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to(Reverse)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Parallel Classifier with Wallaroo")
ab.new_pipeline("Classifier",
wallaroo.TCPSourceConfig(in_host, in_port, decode))
ab.to_stateful(batch_rows, RowBuffer, "CSV rows + global header state")
ab.to_parallel(classify)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode))
return ab.build()
def application_setup(args):
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
gen_source = wallaroo.GenSourceConfig(TransactionsGenerator())
transactions = wallaroo.source("Alerts (stateless)", gen_source)
pipeline = (transactions
.to(check_transaction)
.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode_alert)))
return wallaroo.build_application("Alerts (stateless)", pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Celsius to Fahrenheit")
ab.new_pipeline("Celsius Conversion",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to(Multiply)
ab.to(Add)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
votes = wallaroo.source(
"alphabet", wallaroo.TCPSourceConfig(in_host, in_port, decoder))
pipeline = (votes.key_by(extract_letter).to(add_votes).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("alphabet", pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Word Count Application")
ab.new_pipeline("Split and Count",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to_parallel(split)
ab.to_state_partition(count_word, WordTotal, "word totals", partition)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
gen_source = wallaroo.GenSourceConfig(TransactionsGenerator())
transactions = wallaroo.source("Alerts (windowed)", gen_source)
pipeline = (transactions.key_by(extract_user).to(
wallaroo.range_windows(wallaroo.seconds(9)).with_slide(
wallaroo.seconds(3)).over(TotalAggregation)).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encode_alert)))
return wallaroo.build_application("Alerts (windowed)", pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
letter_partitions = list(string.ascii_lowercase)
ab = wallaroo.ApplicationBuilder("alphabet")
ab.new_pipeline("alphabet",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to_state_partition(AddVotes(), LetterStateBuilder(), "letter state",
LetterPartitionFunction(), letter_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
inputs = wallaroo.source(
"Sequence Window", wallaroo.TCPSourceConfig(in_host, in_port, decoder))
pipeline = (inputs.key_by(constant).to(maybe_one_to_many).key_by(
extract_window).to(observe_new_value).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("Sequence Window", pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Dummy")
ab.new_pipeline("Dummy", wallaroo.TCPSourceConfig(in_host, in_port,
decoder))
ab.to_stateful(count, StateObject, "DummyState")
ab.to_state_partition(count_partitioned, PartitionedStateObject,
"PartitionedDummyState", partition)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
letter_partitions = list(string.ascii_lowercase)
ab = wallaroo.ApplicationBuilder("alphabet")
ab.new_pipeline("alphabet",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to_state_partition(add_votes, TotalVotes, "letter state", partition,
letter_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
global PID
PID=str(os.getpid())
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Parallel Application")
ab.new_pipeline("App",
wallaroo.TCPSourceConfig(in_host, in_port, decode))
ab.to_parallel(classify)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("multi crash dedupe")
ab.new_pipeline("multi crash dedupe",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to(multi)
ab.to(crash_if_none)
ab.to_stateful(dedupe, LastInt, "last int")
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
inputs = wallaroo.source(
"Celsius Conversion",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
pipeline = (inputs.to(multiply).to(add).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args):
global PID
PID = str(os.getpid())
in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
inputs = wallaroo.source(
"App", wallaroo.TCPSourceConfig(in_name, in_host, in_port, decode))
pipeline = (inputs.key_by(extract_key).to(classify).to_sink(
wallaroo.TCPSinkConfig(out_host, out_port, encode)))
return wallaroo.build_application("Parallel App", pipeline)
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
sequence_partitions = ['0', '1']
ab = wallaroo.ApplicationBuilder("Sequence Window")
ab.new_pipeline("Sequence Window",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to(maybe_one_to_many)
ab.to_state_partition(observe_new_value, SequenceWindow, "Sequence Window",
partition, sequence_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
sequence_partitions = [0, 1]
ab = wallaroo.ApplicationBuilder("Sequence Window")
ab.new_pipeline("Sequence Window",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to(MaybeOneToMany)
ab.to_state_partition(ObserveNewValue(),
SequenceWindowStateBuilder(), "Sequence Window",
SequencePartitionFunction(), sequence_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
word_partitions = list(string.ascii_lowercase)
word_partitions.append("!")
ab = wallaroo.ApplicationBuilder("Word Count Application")
ab.new_pipeline("Split and Count",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to_parallel(Split)
ab.to_state_partition(CountWord(), WordTotalsBuilder(), "word totals",
WordPartitionFunction(), word_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
