def application_setup(args): global PID PID=str(os.getpid()) in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] ab = wallaroo.ApplicationBuilder("Parallel Application") ab.new_pipeline("App", wallaroo.TCPSourceConfig(in_host, in_port, decode)) ab.to_parallel(classify) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode)) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] letter_partitions = list(string.ascii_lowercase) ab = wallaroo.ApplicationBuilder("alphabet") ab.new_pipeline("alphabet", wallaroo.TCPSourceConfig(in_host, in_port, Decoder())) ab.to_state_partition(AddVotes(), LetterStateBuilder(), "letter state", LetterPartitionFunction(), letter_partitions) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder())) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] letter_partitions = list(string.ascii_lowercase) ab = wallaroo.ApplicationBuilder("alphabet") ab.new_pipeline("alphabet", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) ab.to_state_partition(add_votes, TotalVotes, "letter state", partition, letter_partitions) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] ab = wallaroo.ApplicationBuilder("multi crash dedupe") ab.new_pipeline("multi crash dedupe", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) ab.to(multi) ab.to(crash_if_none) ab.to_stateful(dedupe, LastInt, "last int") ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] ab = wallaroo.ApplicationBuilder("Dummy") ab.new_pipeline("Dummy", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) ab.to_stateful(count, StateObject, "DummyState") ab.to_state_partition(count_partitioned, PartitionedStateObject, "PartitionedDummyState", partition) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] sequence_partitions = ['0', '1'] ab = wallaroo.ApplicationBuilder("Sequence Window") ab.new_pipeline("Sequence Window", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) ab.to(maybe_one_to_many) ab.to_state_partition(observe_new_value, SequenceWindow, "Sequence Window", partition, sequence_partitions) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] sequence_partitions = [0, 1] ab = wallaroo.ApplicationBuilder("Sequence Window") ab.new_pipeline("Sequence Window", wallaroo.TCPSourceConfig(in_host, in_port, Decoder())) ab.to(MaybeOneToMany) ab.to_state_partition(ObserveNewValue(), SequenceWindowStateBuilder(), "Sequence Window", SequencePartitionFunction(), sequence_partitions) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder())) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] tcp_source = wallaroo.TCPSourceConfig(in_host, in_port, decoder) tcp_sink = wallaroo.TCPSinkConfig(out_host, out_port, encoder) ab = wallaroo.ApplicationBuilder("Spam Detector") ab.new_pipeline("Message text analysis", tcp_source) ab.to(filter_messages_only) ab.to_stateful(update_user_stats, MessagingStatistics, "Count per-user messaging statistics") ab.to_stateful(classify_user, Classifier, "Classify users based on statistics") ab.to_sink(tcp_sink) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] word_partitions = list(string.ascii_lowercase) word_partitions.append("!") ab = wallaroo.ApplicationBuilder("Word Count Application") ab.new_pipeline("Split and Count", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) ab.to_parallel(split) ab.to_state_partition(count_word, WordTotals, "word totals", partition, word_partitions) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return ab.build()
def application_setup(args): parse_delay(args) in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] nonce_source = wallaroo.TCPSourceConfig(in_host, in_port, nonce_decoder) ok_sink = wallaroo.TCPSinkConfig(out_host, out_port, ok_encoder) partitioner = TuplePartitioner(0) ab = wallaroo.ApplicationBuilder("sleepy-python") ab.new_pipeline("Counting Sheep", nonce_source) ab.to(process_nonce) ab.to_state_partition(busy_sleep, DreamData, "dream-data", partitioner, partitioner.partitions) ab.to_sink(ok_sink) return ab.build()
def application_setup(args): ab = wallaroo.ApplicationBuilder("Celsius to Fahrenheit") ab.source_connector("celsius_feed", encoder=encode_feed, decoder=decode_feed) ab.sink_connector("fahrenheit_conversion", encoder=encode_conversion, decoder=decode_conversion) ab.new_pipeline("Celsius Conversion", "celsius_feed") ab.to(multiply) ab.to(add) ab.to_sink("fahrenheit_conversion") return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] raw_hashtag_partitions = list(string.ascii_lowercase) raw_hashtag_partitions.append("!") ab = wallaroo.ApplicationBuilder("Trending Hashtags") ab.new_pipeline("Tweets", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) ab.to_parallel(find_hashtags) ab.to_state_partition(count_hashtags, HashtagCounts, "raw hashtag counts", extract_hashtag_key, raw_hashtag_partitions) ab.to_stateful(top_hashtags, TopTags, "top hashtags") ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] letter_partitions = [a + b for a in lowercase for b in lowercase] letter_partitions.append('!') ab = wallaroo.ApplicationBuilder("alphabet") ab.new_pipeline("alphabet", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) if '--to-parallel' in args: ab.to_parallel(double_vote) ab.to(half_vote) ab.to_state_partition(add_votes, TotalVotes, "letter-state", partition, letter_partitions) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return ab.build()
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] partitions = 10000 server_partitions = range(partitions) ab = wallaroo.ApplicationBuilder("DDoS Attack Detector") ab.new_pipeline("ddos detection", wallaroo.TCPSourceConfig(in_host, in_port, Decoder())) ab.to_state_partition_u64(ProcessLogEntry(), PartitionStatsBuilder(), "process log entry", ServerPartitionFunction(partitions), server_partitions) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder())) return ab.build()
def application_setup(args): global pca global logistic with open('pca.model', 'r') as f: pca = pickle.load(f) with open('logistic.model', 'r') as f: logistic = pickle.load(f) in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] ab = wallaroo.ApplicationBuilder("MNIST classification") ab.new_pipeline("MNIST", wallaroo.TCPSourceConfig(in_host, in_port, decode)) ab.to(pca_transform) ab.to(logistic_classify) ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode)) return ab.build()
def application_setup(args): (in_topic, in_brokers, in_log_level) = wallaroo.kafka_parse_source_options(args) (out_topic, out_brokers, out_log_level, out_max_produce_buffer_ms, out_max_message_size) = wallaroo.kafka_parse_sink_options(args) ab = wallaroo.ApplicationBuilder("Celsius to Fahrenheit with Kafka") ab.new_pipeline("convert", wallaroo.KafkaSourceConfig(in_topic, in_brokers, in_log_level, decoder)) ab.to(multiply) ab.to(add) ab.to_sink(wallaroo.KafkaSinkConfig(out_topic, out_brokers, out_log_level, out_max_produce_buffer_ms, out_max_message_size, encoder)) return ab.build()
def application_setup(args): partition_count = 15 partition_list = list() for x in range(0, partition_count): partition_list.append(x) syncpoint_list = list() syncpoint_list.append("0") ab = wallaroo.ApplicationBuilder("STTWallStatefull") ab.new_pipeline("STTWallStatefull", Decoder()) ab.to_state_partition(Computation_STT(), WordCounterBuilder(), "word counter", RoundRobinPartitionFunction(partition_count), partition_list) #ab.to_state_partition(Computation_SyncPoint(), # SyncPointBuilder(), "sync-point", # SynchornizationPointPartition(), # syncpoint_list) ab.to_sink(Encoder()) return ab.build()
def application_setup(args): # Parse user options for --depth and --internal-source parser = argparse.ArgumentParser("Multi Partition Detector") parser.add_argument("--depth", type=int, default=1, help="The depth of the detector topology") parser.add_argument("--gen-source", action='store_true', help="Use an internal source for resilience tests") parser.add_argument( "--partitions", type=int, default=40, help="Number of partitions for use with internal source") pargs, _ = parser.parse_known_args(args) partitions = [] ab = wallaroo.ApplicationBuilder("Multi Partition Detector") if pargs.gen_source: print("Using internal source generator") source = wallaroo.GenSourceConfig( MultiPartitionGenerator(pargs.partitions)) else: print("Using TCP Source") in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] source = wallaroo.TCPSourceConfig(in_host, in_port, decoder) ab.new_pipeline("Detector", source) for x in range(pargs.depth): ab.to(trace_id) ab.to_state_partition(trace_window, WindowState, "state{}".format(x), partition, partitions) out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return ab.build()
def application_setup(args): """ This is our application's entry point. We read in our command line arguments and describe our processing pipeline. We return a representation of that pipeline which is then started up my the Wallaroo runtime. Keep in mind that this application setup is run both on the primary worker as well as any secondary workers that you add to the cluster. """ source_config = wallaroo.kafka_parse_source_options(args) + (decoder, ) source = wallaroo.KafkaSourceConfig(*source_config) sink_config = wallaroo.kafka_parse_sink_options(args) + (encoder, ) sink = wallaroo.KafkaSinkConfig(*sink_config) ab = wallaroo.ApplicationBuilder("GitHub Star Leaders") ab.new_pipeline("star count leaders", source) ab.to(filter_interesting_events) ab.to_state_partition(annotate_repos, RepoMetadata, "repo_metadata", REPO_PARTITIONER, REPO_PARTITIONER.partitions) ab.to_sink(sink) return ab.build()