def application_setup(args): # Parse user options for --depth and --internal-source parser = argparse.ArgumentParser("Multi Partition Detector") parser.add_argument("--depth", type=int, default=1, help="The depth of the detector topology") parser.add_argument("--gen-source", action='store_true', help="Use an internal source for resilience tests") parser.add_argument("--partitions", type=int, default=40, help="Number of partitions for use with internal source") pargs, _ = parser.parse_known_args(args) if pargs.gen_source: print("Using internal source generator") source = wallaroo.GenSourceConfig(MultiPartitionGenerator(pargs.partitions)) else: print("Using TCP Source") in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] source = wallaroo.TCPSourceConfig(in_host, in_port, decoder) p = wallaroo.source("Detector", source) for x in range(pargs.depth): p = p.key_by(extract_key) p = p.to(trace_id) p = p.key_by(extract_key) p = p.to(trace_window) out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args): values = wallaroo.source("convert", wallaroo.DefaultKafkaSourceCLIParser(decoder)) pipeline = (values.to(multiply).to(add).to_sink( wallaroo.DefaultKafkaSinkCLIParser(encoder))) return wallaroo.build_application("Celsius to Fahrenheit with Kafka", pipeline)
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] pipeline = (wallaroo.source( "Dummy", wallaroo.TCPSourceConfig(in_host, in_port, decoder)).to( count).key_by(partition).to(count_partitioned).to_sink( wallaroo.TCPSinkConfig(out_host, out_port, encoder))) return wallaroo.build_application("Dummy", pipeline)
def application_setup(args): # Parse user options for --depth and --internal-source parser = argparse.ArgumentParser("Multi Partition Detector") parser.add_argument("--depth", type=int, default=1, help="The depth of the detector topology") parser.add_argument( "--source", choices=['tcp', 'gensource', 'alo'], default='tcp', help=("Choose source type for resilience tests. " "'tcp' for standard TCP, 'gensource' for internal " "generator source, and 'alo' for an external at-" "least-once connector source.")) parser.add_argument( "--partitions", type=int, default=40, help="Number of partitions for use with internal source") pargs, _ = parser.parse_known_args(args) if not '--cluster-initializer' in wallaroo._ARGS: pargs.partitions = 0 source_name = "Detector" if pargs.source == 'gensource': print("Using internal source generator") source = wallaroo.GenSourceConfig( source_name, MultiPartitionGenerator(pargs.partitions)) elif pargs.source == 'tcp': print("Using TCP Source") in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] source = wallaroo.TCPSourceConfig(in_name, in_host, in_port, decoder) elif pargs.source == 'alo': print("Using at-least-once source") in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] source = wallaroo.experimental.SourceConnectorConfig( name=source_name, encoder=encode_feed, decoder=decode_feed, host=in_host, port=in_port, cookie="cookie", max_credits=67, refill_credits=10) p = wallaroo.source(source_name, source) for x in range(pargs.depth): p = p.key_by(extract_key) p = p.to(trace_id) p = p.key_by(extract_key) p = p.to(trace_window) out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] lines = wallaroo.source( "Split and Count", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) pipeline = (lines.to(split).key_by(extract_word).to(count_word).to_sink( wallaroo.TCPSinkConfig(out_host, out_port, encoder))) return wallaroo.build_application("Word Count Application", pipeline)
def application_setup(args): input_addrs = wallaroo.tcp_parse_input_addrs(args) order_host, order_port = input_addrs[0] nbbo_host, nbbo_port = input_addrs[1] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] orders = wallaroo.source( "Orders", wallaroo.TCPSourceConfig(order_host, order_port, order_decoder)) market_data = wallaroo.source( "Market Data", wallaroo.TCPSourceConfig(nbbo_host, nbbo_port, market_data_decoder)) pipeline = (orders.merge(market_data).key_by(extract_symbol).to( check_market_data).to_sink( wallaroo.TCPSinkConfig(out_host, out_port, order_result_encoder))) return wallaroo.build_application("Market Spread", pipeline)
def application_setup(args): out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] gen_source = wallaroo.GenSourceConfig(TransactionsGenerator()) transactions = wallaroo.source("Alerts (stateless)", gen_source) pipeline = (transactions .to(check_transaction) .to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode_alert))) return wallaroo.build_application("Alerts (stateless)", pipeline)
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] votes = wallaroo.source( "alphabet", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) pipeline = (votes.key_by(extract_letter).to(add_votes).to_sink( wallaroo.TCPSinkConfig(out_host, out_port, encoder))) return wallaroo.build_application("alphabet", pipeline)
def application_setup(args): celsius_feed = wallaroo.experimental.SourceConnectorConfig( "celsius_feed", encoder=encode_feed, decoder=decode_feed, port=7100) fahrenheit_conversion = wallaroo.experimental.SinkConnectorConfig( "fahrenheit_conversion", encoder=encode_conversion, decoder=decode_conversion, port=7200) pipeline = (wallaroo.source( "convert temperature readings", celsius_feed).to(multiply).to(add).to_sink(fahrenheit_conversion)) return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] inputs = wallaroo.source( "Celsius Conversion", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) pipeline = (inputs.to(multiply).to(add).to_sink( wallaroo.TCPSinkConfig(out_host, out_port, encoder))) return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args): out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] gen_source = wallaroo.GenSourceConfig(TransactionsGenerator()) transactions = wallaroo.source("Alerts (windowed)", gen_source) pipeline = (transactions.key_by(extract_user).to( wallaroo.range_windows(wallaroo.seconds(9)).with_slide( wallaroo.seconds(3)).over(TotalAggregation)).to_sink( wallaroo.TCPSinkConfig(out_host, out_port, encode_alert))) return wallaroo.build_application("Alerts (windowed)", pipeline)
def application_setup(args): in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] inputs = wallaroo.source( "Sequence Window", wallaroo.TCPSourceConfig(in_host, in_port, decoder)) pipeline = (inputs.key_by(constant).to(maybe_one_to_many).key_by( extract_window).to(observe_new_value).to_sink( wallaroo.TCPSinkConfig(out_host, out_port, encoder))) return wallaroo.build_application("Sequence Window", pipeline)
def application_setup(args): input_addrs = wallaroo.tcp_parse_input_addrs(args) inputs = {ip[0]: {"host": ip[1], "port": ip[2]} for ip in input_addrs} out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] orders = wallaroo.source( "Orders", wallaroo.TCPSourceConfig("Orders", inputs["Orders"]["host"], inputs["Orders"]["port"], decode_order)) market_data = wallaroo.source( "Market Data", wallaroo.TCPSourceConfig("Market Data", inputs["Market Data"]["host"], inputs["Market Data"]["port"], decode_market_data)) pipeline = (orders.merge(market_data).key_by(extract_symbol).to( check_market_data).to_sink( wallaroo.TCPSinkConfig(out_host, out_port, encode_order_result))) return wallaroo.build_application("Market Spread", pipeline)
def application_setup(args): global PID PID = str(os.getpid()) in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] inputs = wallaroo.source( "App", wallaroo.TCPSourceConfig(in_name, in_host, in_port, decode)) pipeline = (inputs.key_by(extract_key).to(classify).to_sink( wallaroo.TCPSinkConfig(out_host, out_port, encode))) return wallaroo.build_application("Parallel App", pipeline)
def application_setup(args): celsius_feed = wallaroo.GenSourceConfig('Gen celsius source', CelsiusGenerator(999999)) fahrenheit_conversion = wallaroo.experimental.SinkConnectorConfig( "fahrenheit_conversion", encoder=encode_conversion, decoder=decode_conversion, port=7200, cookie="Dragons Love Tacos!") pipeline = (wallaroo.source( "convert temperature readings", celsius_feed).to(multiply).to(add).to_sink(fahrenheit_conversion)) return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args): parse_delay(args) in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] nonce_source = wallaroo.TCPSourceConfig(in_host, in_port, nonce_decoder) ok_sink = wallaroo.TCPSinkConfig(out_host, out_port, ok_encoder) inputs = wallaroo.source("Counting Sheep", nonce_source) pipeline = (inputs. .to(process_nonce) .key_by(extract_key) .to(busy_sleep) .to_sink(ok_sink)) return wallaroo.build_application("Counting Sheep", pipeline)
def application_setup(args): # Parse user options parser = argparse.ArgumentParser("Topology Test Generator") parser_add_args(parser) pargs, _ = parser.parse_known_args(args) app_name = "topology test" pipe_name = "topology test pipeline" print("Using TCP Source") in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] source = wallaroo.TCPSourceConfig(in_host, in_port, decoder) inputs = wallaroo.source(pipe_name, source) # programmatically add computations topology = Topology(pargs.topology) pipeline = topology.build(inputs) print("Using TCP Sink") out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] pipeline = pipeline.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return wallaroo.build_application(app_name, pipeline)
def application_setup(args): out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] #source_config = wallaroo.TCPSourceConfig("celsius", in_host, in_port, decoder) source_config = wallaroo.experimental.SourceConnectorConfig( "celsius_feed", encoder=encode_feed, decoder=decode_feed, host=in_host, port=in_port, cookie="Dragons Love Tacos!", max_credits=10, refill_credits=8) #sink_config = wallaroo.experimental.SinkConnectorConfig( # "fahrenheit_conversion", # encoder=encode_conversion, # decoder=decode_conversion, # port=7200) sink_config = wallaroo.TCPSinkConfig(out_host, out_port, encode_conversion) pipeline = (wallaroo.source( "convert temperature readings", source_config).to(multiply).to(add).to_sink(sink_config)) return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
def application_setup(args): parser = argparse.ArgumentParser("Window Detector") parser.add_argument("--window-type", default="tumbling", choices=["tumbling", "sliding", "counting"]) parser.add_argument("--window-delay", type=int, default=0, help=("Window delay" "size in milliseconds. (Default: 0)")) parser.add_argument("--window-size", type=int, default=50, help=("Window size in" "milliseconds or units. (Default: 50)")) parser.add_argument("--window-slide", type=int, default=25, help=("Window slide size, in milliseconds. " "(Default: 25)")) parser.add_argument( "--window-late-policy", default="drop", choices=["drop", "fire-per-message", "place-in-oldest-window"]) parser.add_argument( "--source", choices=['tcp', 'gensource', 'alo'], default='tcp', help=("Choose source type for resilience tests. " "'tcp' for standard TCP, 'gensource' for internal " "generator source, and 'alo' for an external at-" "least-once connector source.")) parser.add_argument( "--partitions", type=int, default=40, help="Number of partitions for use with internal source") pargs, _ = parser.parse_known_args(args) if not '--cluster-initializer' in wallaroo._ARGS: pargs.partitions = 0 source_name = "Detector" if pargs.source == 'gensource': print("Using internal source generator") source = wallaroo.GenSourceConfig( source_name, MultiPartitionGenerator(pargs.partitions)) elif pargs.source == 'tcp': print("Using TCP Source") _, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] source = wallaroo.TCPSourceConfig(source_name, in_host, in_port, decoder) elif pargs.source == 'alo': print("Using at-least-once source") _, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] source = wallaroo.experimental.SourceConnectorConfig( name=source_name, encoder=encode_feed, decoder=decode_feed, host=in_host, port=in_port, cookie="cookie", max_credits=67, refill_credits=10) p = wallaroo.source(source_name, source) p = p.key_by(extract_key) p = p.to(trace_id) p = p.key_by(extract_key) # Programmatically construct the window type and arguments if pargs.window_type == 'counting': print("Using window size: {} units".format(pargs.window_size)) window = wallaroo.count_windows(pargs.window_size) else: print("Using window size: {} ms".format(pargs.window_size)) window = wallaroo.range_windows( wallaroo.milliseconds(pargs.window_size)) if pargs.window_delay: print("Using window_delay: {} ms".format(pargs.window_delay)) window = window.with_delay( wallaroo.milliseconds(pargs.window_delay)) if pargs.window_type == 'sliding': print("Using window_slide: {} ms".format(pargs.window_slide)) window = window.with_slide( wallaroo.milliseconds(pargs.window_slide)) if pargs.window_late_policy: ## NOTE: The Wallaroo default policy for late window data is "drop". ## However, it's quite likely that that policy can cause ## intermittent test failures. if pargs.window_late_policy == "drop": print( "\n\nWARNING: drop policy can cause intermittent test failures!\n\n" ) print("Using window_late_policy: {}".format( pargs.window_late_policy)) window = window.with_late_data_policy(pargs.window_late_policy) # add the window to the topology p = p.to(window.over(Collect)) p = p.to(split_accumulated) out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return wallaroo.build_application("Tumbling Time Window Detector", p)
def application_setup(args): parser = argparse.ArgumentParser("Window Detector") parser.add_argument("--window-type", default="tumbling", choices=["tumbling", "sliding", "counting"]) parser.add_argument("--window-delay", type=int, default=0, help=("Window delay" "size in milliseconds. (Default: 0)")) parser.add_argument("--window-size", type=int, default=50, help=("Window size in" "milliseconds or units. (Default: 50)")) parser.add_argument("--window-slide", type=int, default=25, help=("Window slide size, in milliseconds. " "(Default: 25)")) parser.add_argument("--gen-source", action='store_true', help="Use an internal source for resilience tests") parser.add_argument( "--partitions", type=int, default=40, help="Number of partitions for use with internal source") pargs, _ = parser.parse_known_args(args) if pargs.gen_source: print("Using internal source generator") source = wallaroo.GenSourceConfig( MultiPartitionGenerator(pargs.partitions)) else: print("Using TCP Source") in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0] source = wallaroo.TCPSourceConfig(in_host, in_port, decoder) p = wallaroo.source("{} window".format(pargs.window_type), source) p = p.key_by(extract_key) p = p.to(trace_id) p = p.key_by(extract_key) # Programmatically construct the window type and arguments if pargs.window_type == 'counting': print("Using window size: {} units".format(pargs.window_size)) window = wallaroo.count_windows(pargs.window_size) else: print("Using window size: {} ms".format(pargs.window_size)) window = wallaroo.range_windows( wallaroo.milliseconds(pargs.window_size)) if pargs.window_delay: print("Using window_delay: {} ms".format(pargs.window_delay)) window = window.with_delay( wallaroo.milliseconds(pargs.window_delay)) if pargs.window_type == 'sliding': print("Using window_slide: {} ms".format(pargs.window_slide)) window = window.with_slide( wallaroo.milliseconds(pargs.window_slide)) # add the window to the topology p = p.to(window.over(Collect)) p = p.to(split_accumulated) out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0] p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder)) return wallaroo.build_application("Tumbling Time Window Detector", p)