def application_setup(args):
global PID
PID=str(os.getpid())
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Parallel Application")
ab.new_pipeline("App",
wallaroo.TCPSourceConfig(in_host, in_port, decode))
ab.to_parallel(classify)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
letter_partitions = list(string.ascii_lowercase)
ab = wallaroo.ApplicationBuilder("alphabet")
ab.new_pipeline("alphabet",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to_state_partition(AddVotes(), LetterStateBuilder(), "letter state",
LetterPartitionFunction(), letter_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
letter_partitions = list(string.ascii_lowercase)
ab = wallaroo.ApplicationBuilder("alphabet")
ab.new_pipeline("alphabet",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to_state_partition(add_votes, TotalVotes, "letter state", partition,
letter_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("multi crash dedupe")
ab.new_pipeline("multi crash dedupe",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to(multi)
ab.to(crash_if_none)
ab.to_stateful(dedupe, LastInt, "last int")
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("Dummy")
ab.new_pipeline("Dummy", wallaroo.TCPSourceConfig(in_host, in_port,
decoder))
ab.to_stateful(count, StateObject, "DummyState")
ab.to_state_partition(count_partitioned, PartitionedStateObject,
"PartitionedDummyState", partition)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
sequence_partitions = ['0', '1']
ab = wallaroo.ApplicationBuilder("Sequence Window")
ab.new_pipeline("Sequence Window",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to(maybe_one_to_many)
ab.to_state_partition(observe_new_value, SequenceWindow, "Sequence Window",
partition, sequence_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
sequence_partitions = [0, 1]
ab = wallaroo.ApplicationBuilder("Sequence Window")
ab.new_pipeline("Sequence Window",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to(MaybeOneToMany)
ab.to_state_partition(ObserveNewValue(),
SequenceWindowStateBuilder(), "Sequence Window",
SequencePartitionFunction(), sequence_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
tcp_source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)
tcp_sink = wallaroo.TCPSinkConfig(out_host, out_port, encoder)
ab = wallaroo.ApplicationBuilder("Spam Detector")
ab.new_pipeline("Message text analysis", tcp_source)
ab.to(filter_messages_only)
ab.to_stateful(update_user_stats, MessagingStatistics,
"Count per-user messaging statistics")
ab.to_stateful(classify_user, Classifier,
"Classify users based on statistics")
ab.to_sink(tcp_sink)
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
word_partitions = list(string.ascii_lowercase)
word_partitions.append("!")
ab = wallaroo.ApplicationBuilder("Word Count Application")
ab.new_pipeline("Split and Count",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to_parallel(split)
ab.to_state_partition(count_word, WordTotals, "word totals",
partition, word_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
parse_delay(args)
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
nonce_source = wallaroo.TCPSourceConfig(in_host, in_port, nonce_decoder)
ok_sink = wallaroo.TCPSinkConfig(out_host, out_port, ok_encoder)
partitioner = TuplePartitioner(0)
ab = wallaroo.ApplicationBuilder("sleepy-python")
ab.new_pipeline("Counting Sheep", nonce_source)
ab.to(process_nonce)
ab.to_state_partition(busy_sleep, DreamData, "dream-data", partitioner,
partitioner.partitions)
ab.to_sink(ok_sink)
return ab.build()
def application_setup(args):
ab = wallaroo.ApplicationBuilder("Celsius to Fahrenheit")
ab.source_connector("celsius_feed",
encoder=encode_feed,
decoder=decode_feed)
ab.sink_connector("fahrenheit_conversion",
encoder=encode_conversion,
decoder=decode_conversion)
ab.new_pipeline("Celsius Conversion", "celsius_feed")
ab.to(multiply)
ab.to(add)
ab.to_sink("fahrenheit_conversion")
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
raw_hashtag_partitions = list(string.ascii_lowercase)
raw_hashtag_partitions.append("!")
ab = wallaroo.ApplicationBuilder("Trending Hashtags")
ab.new_pipeline("Tweets",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
ab.to_parallel(find_hashtags)
ab.to_state_partition(count_hashtags, HashtagCounts, "raw hashtag counts",
extract_hashtag_key, raw_hashtag_partitions)
ab.to_stateful(top_hashtags, TopTags, "top hashtags")
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
letter_partitions = [a + b for a in lowercase for b in lowercase]
letter_partitions.append('!')
ab = wallaroo.ApplicationBuilder("alphabet")
ab.new_pipeline("alphabet",
wallaroo.TCPSourceConfig(in_host, in_port, decoder))
if '--to-parallel' in args:
ab.to_parallel(double_vote)
ab.to(half_vote)
ab.to_state_partition(add_votes, TotalVotes, "letter-state",
partition, letter_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
partitions = 10000
server_partitions = range(partitions)
ab = wallaroo.ApplicationBuilder("DDoS Attack Detector")
ab.new_pipeline("ddos detection",
wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
ab.to_state_partition_u64(ProcessLogEntry(), PartitionStatsBuilder(),
"process log entry",
ServerPartitionFunction(partitions),
server_partitions)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
return ab.build()
def application_setup(args):
global pca
global logistic
with open('pca.model', 'r') as f:
pca = pickle.load(f)
with open('logistic.model', 'r') as f:
logistic = pickle.load(f)
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab = wallaroo.ApplicationBuilder("MNIST classification")
ab.new_pipeline("MNIST", wallaroo.TCPSourceConfig(in_host, in_port,
decode))
ab.to(pca_transform)
ab.to(logistic_classify)
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode))
return ab.build()
def application_setup(args):
(in_topic, in_brokers,
in_log_level) = wallaroo.kafka_parse_source_options(args)
(out_topic, out_brokers, out_log_level, out_max_produce_buffer_ms,
out_max_message_size) = wallaroo.kafka_parse_sink_options(args)
ab = wallaroo.ApplicationBuilder("Celsius to Fahrenheit with Kafka")
ab.new_pipeline("convert",
wallaroo.KafkaSourceConfig(in_topic, in_brokers, in_log_level,
decoder))
ab.to(multiply)
ab.to(add)
ab.to_sink(wallaroo.KafkaSinkConfig(out_topic, out_brokers, out_log_level,
out_max_produce_buffer_ms,
out_max_message_size,
encoder))
return ab.build()
def application_setup(args):
partition_count = 15
partition_list = list()
for x in range(0, partition_count):
partition_list.append(x)
syncpoint_list = list()
syncpoint_list.append("0")
ab = wallaroo.ApplicationBuilder("STTWallStatefull")
ab.new_pipeline("STTWallStatefull", Decoder())
ab.to_state_partition(Computation_STT(), WordCounterBuilder(),
"word counter",
RoundRobinPartitionFunction(partition_count),
partition_list)
#ab.to_state_partition(Computation_SyncPoint(),
# SyncPointBuilder(), "sync-point",
# SynchornizationPointPartition(),
# syncpoint_list)
ab.to_sink(Encoder())
return ab.build()
def application_setup(args):
# Parse user options for --depth and --internal-source
parser = argparse.ArgumentParser("Multi Partition Detector")
parser.add_argument("--depth",
type=int,
default=1,
help="The depth of the detector topology")
parser.add_argument("--gen-source",
action='store_true',
help="Use an internal source for resilience tests")
parser.add_argument(
"--partitions",
type=int,
default=40,
help="Number of partitions for use with internal source")
pargs, _ = parser.parse_known_args(args)
partitions = []
ab = wallaroo.ApplicationBuilder("Multi Partition Detector")
if pargs.gen_source:
print("Using internal source generator")
source = wallaroo.GenSourceConfig(
MultiPartitionGenerator(pargs.partitions))
else:
print("Using TCP Source")
in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)
ab.new_pipeline("Detector", source)
for x in range(pargs.depth):
ab.to(trace_id)
ab.to_state_partition(trace_window, WindowState, "state{}".format(x),
partition, partitions)
out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
return ab.build()
def application_setup(args):
"""
This is our application's entry point. We read in our
command line arguments and describe our processing
pipeline. We return a representation of that pipeline which
is then started up my the Wallaroo runtime. Keep in mind
that this application setup is run both on the primary
worker as well as any secondary workers that you add to the
cluster.
"""
source_config = wallaroo.kafka_parse_source_options(args) + (decoder, )
source = wallaroo.KafkaSourceConfig(*source_config)
sink_config = wallaroo.kafka_parse_sink_options(args) + (encoder, )
sink = wallaroo.KafkaSinkConfig(*sink_config)
ab = wallaroo.ApplicationBuilder("GitHub Star Leaders")
ab.new_pipeline("star count leaders", source)
ab.to(filter_interesting_events)
ab.to_state_partition(annotate_repos, RepoMetadata, "repo_metadata",
REPO_PARTITIONER, REPO_PARTITIONER.partitions)
ab.to_sink(sink)
return ab.build()
