示例#1
0
def application_setup(args):
    input_addrs = wallaroo.tcp_parse_input_addrs(args)
    order_host, order_port = input_addrs[0]
    nbbo_host, nbbo_port = input_addrs[1]

    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    symbol_partitions = [
        str_to_partition(x.rjust(4)) for x in load_valid_symbols()
    ]

    ab = wallaroo.ApplicationBuilder("market-spread")
    ab.new_pipeline(
        "Orders",
        wallaroo.TCPSourceConfig(
            order_host, order_port, OrderDecoder())).to_state_partition_u64(
                CheckOrder(), SymbolDataBuilder(), "symbol-data",
                SymbolPartitionFunction(), symbol_partitions).to_sink(
                    wallaroo.TCPSinkConfig(
                        out_host, out_port,
                        OrderResultEncoder())).new_pipeline(
                            "Market Data",
                            wallaroo.TCPSourceConfig(
                                nbbo_host, nbbo_port,
                                MarketDataDecoder())).to_state_partition_u64(
                                    UpdateMarketData(), SymbolDataBuilder(),
                                    "symbol-data", SymbolPartitionFunction(),
                                    symbol_partitions).done()
    return ab.build()
示例#2
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
    ll_host, ll_port = wallaroo.tcp_parse_input_addrs(args)[1]
    ab = wallaroo.ApplicationBuilder("gdax alerts")
    # Pipeline to store alerts and the corresponding user
    ab.new_pipeline("Load users created alerts",
                    wallaroo.TCPSourceConfig(ll_host, ll_port, decoder))
    ab.to_stateful(save_alert, Alerts, "user set alerts")
    ab.done()

    # Pipeline to calculate current bitcoin price from gdax
    ab.new_pipeline("Average BTC price",
                    wallaroo.TCPSourceConfig(in_host, in_port, decoder))
    ab.to_parallel(extract_price)
    # State object, current_price
    # Update the moving
    ab.to_stateful(calculate_and_update_average_price, BTCPrice,
                   "Calculate and update the average price")
    # Alerts are map of `price: [user_ids]`
    # Alert.update lookup users, return and replace with None
    # Send to another application, that then calls a cellery task
    ab.to_stateful(maybe_send_alerts_based_on_average_price, Alerts,
                   "user set alerts")
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return ab.build()
示例#3
0
def application_setup(args):
    # Parse user options for --depth and --internal-source
    parser = argparse.ArgumentParser("Multi Partition Detector")
    parser.add_argument("--depth", type=int, default=1,
                    help="The depth of the detector topology")
    parser.add_argument("--gen-source", action='store_true',
                    help="Use an internal source for resilience tests")
    parser.add_argument("--partitions", type=int, default=40,
                    help="Number of partitions for use with internal source")
    pargs, _ = parser.parse_known_args(args)

    if pargs.gen_source:
        print("Using internal source generator")
        source = wallaroo.GenSourceConfig(MultiPartitionGenerator(pargs.partitions))
    else:
        print("Using TCP Source")
        in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
        source = wallaroo.TCPSourceConfig(in_host, in_port, decoder)

    p = wallaroo.source("Detector", source)
    for x in range(pargs.depth):
        p = p.key_by(extract_key)
        p = p.to(trace_id)
        p = p.key_by(extract_key)
        p = p.to(trace_window)

    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
    p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args):
    input_addrs = wallaroo.tcp_parse_input_addrs(args)

    ll_host, ll_port = input_addrs[0]
    cc_host, cc_port = input_addrs[1]

    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    initial_partitions = range(0, 10)

    ab = wallaroo.ApplicationBuilder("Ad Tech Application")
    ab.new_pipeline("Load loyalty customers",
                    wallaroo.TCPSourceConfig(ll_host, ll_port, ll_decoder))
    ab.to_state_partition(save_customer, LoyaltyCustomers, "loyalty customers",
                          extract_conversion_key, initial_partitions)
    ab.done()

    ab.new_pipeline("Conversions",
                    wallaroo.TCPSourceConfig(cc_host, cc_port, cc_decoder))
    ab.to_state_partition(process_email_add_customer, LoyaltyCustomers,
                          "loyalty customers", extract_conversion_key,
                          initial_partitions)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, cc_encoder))

    return ab.build()
示例#5
0
def application_setup(args):
    input_addrs = wallaroo.tcp_parse_input_addrs(args)
    order_host, order_port = input_addrs[0]
    nbbo_host, nbbo_port = input_addrs[1]

    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    symbol_partitions = [str_to_partition(x.rjust(4)) for x in
                         load_valid_symbols()]

    ab = wallaroo.ApplicationBuilder("market-spread")
    ab.new_pipeline(
            "Orders",
            wallaroo.TCPSourceConfig(order_host, order_port, order_decoder)
        ).to_state_partition_u64(
            check_order, SymbolData, "symbol-data",
            symbol_partition_function, symbol_partitions
        ).to_sink(wallaroo.TCPSinkConfig(out_host, out_port,
                                        order_result_encoder)
        ).new_pipeline(
            "Market Data",
            wallaroo.TCPSourceConfig(nbbo_host, nbbo_port,
                                     market_data_decoder)
        ).to_state_partition_u64(
            update_market_data, SymbolData, "symbol-data",
            symbol_partition_function, symbol_partitions
        ).done()
    return ab.build()
示例#6
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    pipeline = (wallaroo.source(
        "Dummy", wallaroo.TCPSourceConfig(in_host, in_port, decoder)).to(
            count).key_by(partition).to(count_partitioned).to_sink(
                wallaroo.TCPSinkConfig(out_host, out_port, encoder)))
    return wallaroo.build_application("Dummy", pipeline)
示例#7
0
def application_setup(args):
    # Parse user options for --depth and --internal-source
    parser = argparse.ArgumentParser("Multi Partition Detector")
    parser.add_argument("--depth",
                        type=int,
                        default=1,
                        help="The depth of the detector topology")
    parser.add_argument(
        "--source",
        choices=['tcp', 'gensource', 'alo'],
        default='tcp',
        help=("Choose source type for resilience tests. "
              "'tcp' for standard TCP, 'gensource' for internal "
              "generator source, and 'alo' for an external at-"
              "least-once connector source."))
    parser.add_argument(
        "--partitions",
        type=int,
        default=40,
        help="Number of partitions for use with internal source")
    pargs, _ = parser.parse_known_args(args)

    if not '--cluster-initializer' in wallaroo._ARGS:
        pargs.partitions = 0

    source_name = "Detector"
    if pargs.source == 'gensource':
        print("Using internal source generator")
        source = wallaroo.GenSourceConfig(
            source_name, MultiPartitionGenerator(pargs.partitions))
    elif pargs.source == 'tcp':
        print("Using TCP Source")
        in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
        source = wallaroo.TCPSourceConfig(in_name, in_host, in_port, decoder)
    elif pargs.source == 'alo':
        print("Using at-least-once source")
        in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
        source = wallaroo.experimental.SourceConnectorConfig(
            name=source_name,
            encoder=encode_feed,
            decoder=decode_feed,
            host=in_host,
            port=in_port,
            cookie="cookie",
            max_credits=67,
            refill_credits=10)

    p = wallaroo.source(source_name, source)
    for x in range(pargs.depth):
        p = p.key_by(extract_key)
        p = p.to(trace_id)
        p = p.key_by(extract_key)
        p = p.to(trace_window)

    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
    p = p.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return wallaroo.build_application("Multi Partition Detector", p)
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]
    ab = wallaroo.ApplicationBuilder("Trending Hashtags")
    ab.new_pipeline("Tweets_new",
                    wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
    ab.to(HashtagFinder)
    ab.to_stateful(ComputeHashtags(), HashtagsStateBuilder(), "hashtags state")
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
    return ab.build()
示例#9
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("Add2")
    ab.new_pipeline("add_2",
                    wallaroo.TCPSourceConfig(in_host, in_port, decoder))
    ab.to(add_2)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return ab.build()
示例#10
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("alphabet")
    ab.new_pipeline("alphabet",
                    wallaroo.TCPSourceConfig(in_host, in_port, decoder))
    ab.to_stateful(add_votes, AllVotes, "letter state")
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return ab.build()
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("Apache Log file analysis")
    ab.new_pipeline("convert",
                    wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
    ab.to_stateful(Count(), CounterBuilder(), "status counter")
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
    return ab.build()
示例#12
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    lines = wallaroo.source(
        "Split and Count", wallaroo.TCPSourceConfig(in_host, in_port, decoder))
    pipeline = (lines.to(split).key_by(extract_word).to(count_word).to_sink(
        wallaroo.TCPSinkConfig(out_host, out_port, encoder)))

    return wallaroo.build_application("Word Count Application", pipeline)
示例#13
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("Reverse Word")
    ab.new_pipeline("reverse",
                    wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
    ab.to(Reverse)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
    return ab.build()
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("Parallel Classifier with Wallaroo")
    ab.new_pipeline("Classifier",
                    wallaroo.TCPSourceConfig(in_host, in_port, decode))
    ab.to_stateful(batch_rows, RowBuffer, "CSV rows + global header state")
    ab.to_parallel(classify)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode))
    return ab.build()
示例#15
0
def application_setup(args):
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    gen_source = wallaroo.GenSourceConfig(TransactionsGenerator())

    transactions = wallaroo.source("Alerts (stateless)", gen_source)
    pipeline = (transactions
        .to(check_transaction)
        .to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode_alert)))

    return wallaroo.build_application("Alerts (stateless)", pipeline)
示例#16
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("Celsius to Fahrenheit")
    ab.new_pipeline("Celsius Conversion",
                    wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
    ab.to(Multiply)
    ab.to(Add)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
    return ab.build()
示例#17
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    votes = wallaroo.source(
        "alphabet", wallaroo.TCPSourceConfig(in_host, in_port, decoder))

    pipeline = (votes.key_by(extract_letter).to(add_votes).to_sink(
        wallaroo.TCPSinkConfig(out_host, out_port, encoder)))

    return wallaroo.build_application("alphabet", pipeline)
示例#18
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("Word Count Application")
    ab.new_pipeline("Split and Count",
                    wallaroo.TCPSourceConfig(in_host, in_port, decoder))
    ab.to_parallel(split)
    ab.to_state_partition(count_word, WordTotal, "word totals", partition)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return ab.build()
示例#19
0
def application_setup(args):
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    gen_source = wallaroo.GenSourceConfig(TransactionsGenerator())

    transactions = wallaroo.source("Alerts (windowed)", gen_source)
    pipeline = (transactions.key_by(extract_user).to(
        wallaroo.range_windows(wallaroo.seconds(9)).with_slide(
            wallaroo.seconds(3)).over(TotalAggregation)).to_sink(
                wallaroo.TCPSinkConfig(out_host, out_port, encode_alert)))

    return wallaroo.build_application("Alerts (windowed)", pipeline)
示例#20
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    letter_partitions = list(string.ascii_lowercase)
    ab = wallaroo.ApplicationBuilder("alphabet")
    ab.new_pipeline("alphabet",
                    wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
    ab.to_state_partition(AddVotes(), LetterStateBuilder(), "letter state",
                          LetterPartitionFunction(), letter_partitions)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
    return ab.build()
示例#21
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    inputs = wallaroo.source(
        "Sequence Window", wallaroo.TCPSourceConfig(in_host, in_port, decoder))

    pipeline = (inputs.key_by(constant).to(maybe_one_to_many).key_by(
        extract_window).to(observe_new_value).to_sink(
            wallaroo.TCPSinkConfig(out_host, out_port, encoder)))

    return wallaroo.build_application("Sequence Window", pipeline)
示例#22
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("Dummy")
    ab.new_pipeline("Dummy", wallaroo.TCPSourceConfig(in_host, in_port,
                                                      decoder))
    ab.to_stateful(count, StateObject, "DummyState")
    ab.to_state_partition(count_partitioned, PartitionedStateObject,
                          "PartitionedDummyState", partition)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return ab.build()
示例#23
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    letter_partitions = list(string.ascii_lowercase)
    ab = wallaroo.ApplicationBuilder("alphabet")
    ab.new_pipeline("alphabet",
                    wallaroo.TCPSourceConfig(in_host, in_port, decoder))
    ab.to_state_partition(add_votes, TotalVotes, "letter state", partition,
                          letter_partitions)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return ab.build()
示例#24
0
def application_setup(args):
    global PID
    PID=str(os.getpid())
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("Parallel Application")
    ab.new_pipeline("App",
                    wallaroo.TCPSourceConfig(in_host, in_port, decode))
    ab.to_parallel(classify)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encode))
    return ab.build()
示例#25
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    ab = wallaroo.ApplicationBuilder("multi crash dedupe")
    ab.new_pipeline("multi crash dedupe",
                    wallaroo.TCPSourceConfig(in_host, in_port, decoder))
    ab.to(multi)
    ab.to(crash_if_none)
    ab.to_stateful(dedupe, LastInt, "last int")
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return ab.build()
示例#26
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    inputs = wallaroo.source(
        "Celsius Conversion",
        wallaroo.TCPSourceConfig(in_host, in_port, decoder))

    pipeline = (inputs.to(multiply).to(add).to_sink(
        wallaroo.TCPSinkConfig(out_host, out_port, encoder)))

    return wallaroo.build_application("Celsius to Fahrenheit", pipeline)
示例#27
0
def application_setup(args):
    global PID
    PID = str(os.getpid())
    in_name, in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    inputs = wallaroo.source(
        "App", wallaroo.TCPSourceConfig(in_name, in_host, in_port, decode))

    pipeline = (inputs.key_by(extract_key).to(classify).to_sink(
        wallaroo.TCPSinkConfig(out_host, out_port, encode)))

    return wallaroo.build_application("Parallel App", pipeline)
示例#28
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    sequence_partitions = ['0', '1']
    ab = wallaroo.ApplicationBuilder("Sequence Window")
    ab.new_pipeline("Sequence Window",
                    wallaroo.TCPSourceConfig(in_host, in_port, decoder))
    ab.to(maybe_one_to_many)
    ab.to_state_partition(observe_new_value, SequenceWindow, "Sequence Window",
                          partition, sequence_partitions)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, encoder))
    return ab.build()
示例#29
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    sequence_partitions = [0, 1]
    ab = wallaroo.ApplicationBuilder("Sequence Window")
    ab.new_pipeline("Sequence Window",
                    wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
    ab.to(MaybeOneToMany)
    ab.to_state_partition(ObserveNewValue(),
                          SequenceWindowStateBuilder(), "Sequence Window",
                          SequencePartitionFunction(), sequence_partitions)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
    return ab.build()
示例#30
0
def application_setup(args):
    in_host, in_port = wallaroo.tcp_parse_input_addrs(args)[0]
    out_host, out_port = wallaroo.tcp_parse_output_addrs(args)[0]

    word_partitions = list(string.ascii_lowercase)
    word_partitions.append("!")

    ab = wallaroo.ApplicationBuilder("Word Count Application")
    ab.new_pipeline("Split and Count",
                    wallaroo.TCPSourceConfig(in_host, in_port, Decoder()))
    ab.to_parallel(Split)
    ab.to_state_partition(CountWord(), WordTotalsBuilder(), "word totals",
        WordPartitionFunction(), word_partitions)
    ab.to_sink(wallaroo.TCPSinkConfig(out_host, out_port, Encoder()))
    return ab.build()