def load(events, metadata=None): # find winning bids for each closed auction return (events # find winning bids | beam.Filter(nexmark_query_util.auction_or_bid) | winning_bids.WinningBids() # (auction_bids -> (aution.seller, bid) | beam.Map(lambda auc_bid: (auc_bid.auction.seller, auc_bid.bid)) # calculate and output mean as data arrives | beam.WindowInto( window.GlobalWindows(), trigger=trigger.Repeatedly(trigger.AfterCount(1)), accumulation_mode=trigger.AccumulationMode.ACCUMULATING, allowed_lateness=0) | beam.CombinePerKey(MovingMeanSellingPriceFn(10)) | beam.Map(lambda t: { ResultNames.SELLER: t[0], ResultNames.PRICE: t[1] }))
def load(events, metadata=None): # find winning bids for each closed auction all_winning_bids = (events | beam.Filter(nexmark_query_util.auction_or_bid) | winning_bids.WinningBids()) return ( all_winning_bids # key winning bids by auction category | beam.Map(lambda auc_bid: (auc_bid.auction.category, auc_bid.bid.price)) # re-window for sliding average | beam.WindowInto( window.SlidingWindows(metadata.get('window_size_sec'), metadata.get('window_period_sec'))) # average for each category | beam.CombinePerKey(beam.combiners.MeanCombineFn()) # TODO(leiyiz): fanout with sliding window produces duplicated results, # uncomment after it is fixed [BEAM-10617] # .with_hot_key_fanout(metadata.get('fanout')) # produce output | beam.ParDo(ProjectToCategoryPriceFn()))
def load(events, metadata=None): return (events | beam.Filter(nexmark_query_util.auction_or_bid) | winning_bids.WinningBids())