def observe_market(observer): class FakeQueue: def put(self, value): observer.on_next(value) listener = betfairlightweight.StreamListener( output_queue=FakeQueue(), lightweight=True) self._market_stream = self._client.streaming.create_stream( listener=listener, description='BFG Market Stream') market_filter = streaming_market_filter(market_ids=[ market.marketId for market in self.todays_racecard ]) market_data_filter = streaming_market_data_filter( fields=[ 'EX_BEST_OFFERS', 'EX_TRADED', 'EX_TRADED_VOL', 'EX_LTP', 'EX_MARKET_DEF' ], ladder_levels=3, # Market depth ) self._market_stream.subscribe_to_markets( market_filter=market_filter, market_data_filter=market_data_filter, # conflate_ms=1000, ) self._market_stream.start(async=True)
def __init__( self, storage_engine, market_filter=None, market_data_filter=None, local_dir=LOCAL_DIR, ): self.storage_engine = storage_engine self.storage_engine.recorder = self self.market_filter = market_filter or streaming_market_filter() self.market_data_filter = market_data_filter or streaming_market_data_filter( fields=[ "EX_ALL_OFFERS", "EX_TRADED", "EX_TRADED_VOL", "EX_LTP", "EX_MARKET_DEF", "SP_TRADED", "SP_PROJECTED", ] ) self.local_dir = local_dir self.stream_id = ( create_short_uuid() ) # used to differentiate markets /<self.local_dir>/<stream_id> self.live_markets = [] # list of markets to be processed self._setup() logger.info("Recorder created %s" % self.stream_id)
def markets(filter_path, stream_name): if filter_path is None: return with open(filter_path, "r") as f: print("opening market filter file ... ") s = json.load(f) mf = streaming_market_filter(**s['market_filter']) mdf = streaming_market_data_filter(**s['market_data_filter']) print(sub(stream_name, subscription_message(mf, mdf)))
def __init__(self, storage_engine, market_filter=None, market_data_filter=None): self.storage_engine = storage_engine self.market_filter = market_filter or streaming_market_filter() self.market_data_filter = market_data_filter or streaming_market_data_filter( fields=[ 'EX_ALL_OFFERS', 'EX_TRADED', 'EX_TRADED_VOL', 'EX_LTP', 'EX_MARKET_DEF', 'SP_TRADED', 'SP_PROJECTED' ] ) self.stream_id = create_short_uuid() # used to differentiate markets /<FLUMINE_DATA>/<stream_id> self.live_markets = [] # list of markets to be processed self._setup() logger.info('Recorder created %s' % self.stream_id)
def start_betfair_ladder_stream(self, market_id: str, conflate_ms: float) -> queue.Queue: """Start the Betfair ladder stream. Args: market_id: Betfair market ID. conflate_ms: Conflation rate in milliseconds. Returns: Market ladder queue. """ if self.stream is not None: logger.info( "There is already a Betfair market stream running. Before " "starting a new stream, the existing one must be stopped.") ladder_queue = self.stream.listener.output_queue return ladder_queue logger.info("Initialising output queue.") ladder_queue = queue.Queue() logger.info("Initialising Betfair stream listener.") listener = betfairlightweight.StreamListener(ladder_queue) logger.info("Creating the Betfair market stream.") stream = self._client.streaming.create_stream(listener=listener) logger.info("Setting the market filter to market_id=%s.", market_id) market_filter_ = streaming_market_filter(market_ids=[market_id]) logger.info("Initialising streaming market data filter.") market_data_filter = streaming_market_data_filter( # fields=['EX_MARKET_DEF', 'EX_ALL_OFFERS'], # Without virtual bets fields=['EX_MARKET_DEF', 'EX_BEST_OFFERS_DISP'], # With virtual bets ladder_levels=10) logger.info("Subscribing to the market.") stream.subscribe_to_markets( market_filter=market_filter_, market_data_filter=market_data_filter, conflate_ms=min(conflate_ms, 120000), ) logger.info("Starting the Betfair market stream.") stream.start(async_=True) self.stream = stream return ladder_queue
def create_stream(self, output_queue, market_ids): """ Create Betfair stream that allow to establish connection once and receive MarketBook updates in real time """ # create stream listener listener = betfairlightweight.StreamListener(output_queue=output_queue) # create stream stream = self.api.streaming.create_stream(listener=listener) market_filter = filters.streaming_market_filter(market_ids=market_ids) market_data_filter = filters.streaming_market_data_filter( fields=["EX_BEST_OFFERS", "EX_MARKET_DEF"], ladder_levels=3) # subscribe streaming_unique_id = stream.subscribe_to_markets( market_filter=market_filter, market_data_filter=market_data_filter, conflate_ms=2000, # send update every 1000ms ) return stream
order_type=LimitOrder(price=1.01, size=selection["liability"]), ) market.place_order(order) trading = betfairlightweight.APIClient("username") client = clients.BetfairClient(trading) framework = Flumine(client=client) strategy = SingleStrategy( name="back_strat_42", market_filter=streaming_market_filter( event_type_ids=["7"], country_codes=["GB", "IE"], market_types=["WIN"], ), market_data_filter=streaming_market_data_filter( fields=[ "EX_BEST_OFFERS", "EX_LTP", "EX_MARKET_DEF", ], ladder_levels=1, ), conflate_ms=1000, # update every 1s max_trade_count=1, # 1 trade/order per selection only context={ "selections": [{ "market_id": "1.196154851",
def data_collection_pipeline() -> str: """Pipeline to collect Betfair odds market ladder streaming data. Returns: Path of the output CSV file. """ logging.basicConfig( format='%(levelname)-8s | %(asctime)s | %(name)s: %(message)s', level=logging.INFO) market_id, output_dir, conflate_ms, no_virtual_bets, allow_inplay,\ mins_before_start = parse_command_line_args() trading = bfl.APIClient(username=username, password=password, app_key=app_key, cert_files=[cert_file, cert_key_file]) logger.info("Logging in to Betfair") trading.login() # Event and market information event_type, event, competition = get_event_info(trading, market_id) market_name, market_start_time, selections = get_market_info( trading, market_id) # Wait to stream until a certain amount of minutes before the start if mins_before_start is not None: logger.info( "Logging off from Betfair and waiting until %s minutes before the " "start of the event. Press Ctrl+C to quit.", mins_before_start) trading.logout() now = datetime.utcnow() try: while market_start_time - now >= \ timedelta(minutes=mins_before_start): time.sleep(1) now = datetime.utcnow() except KeyboardInterrupt: logger.info("Exiting program (Keyboard interrupt)") exit(0) logger.info("Logging in to Betfair again.") trading.login() # Output file path output_file_name = get_output_file_name(event_type, event, competition, market_name, market_start_time) output_sqlite_file = os.path.join(output_dir, output_file_name + '.db') output_zip_file = os.path.join(output_dir, output_file_name + '.zip') # Market stream logger.info("Initialising output queue") output_queue = queue.Queue() logger.info("Initialising Betfair stream listener") listener = bfl.StreamListener(output_queue) logger.info("Creating the Betfair market stream") stream = trading.streaming.create_stream(listener=listener) logger.info("Setting the market filter to market_id=%s", market_id) market_filter = streaming_market_filter(market_ids=[market_id]) logger.info("Initialising streaming market data filter") if no_virtual_bets: market_data_fields = ['EX_MARKET_DEF', 'EX_ALL_OFFERS', 'EX_TRADED'] else: market_data_fields = [ 'EX_MARKET_DEF', 'EX_BEST_OFFERS_DISP', 'EX_TRADED' ] market_data_filter = streaming_market_data_filter( fields=market_data_fields, ) logger.info("Subscribing to the market") stream.subscribe_to_markets(market_filter=market_filter, market_data_filter=market_data_filter, conflate_ms=conflate_ms) logger.info("Starting the stream") stream.start(async_=True) logger.info(f"Saving data in file {output_sqlite_file}") connection, cursor = create_sqlite_database(output_sqlite_file) market_snapshot_no = 0 while True: try: market_books = output_queue.get() market_book = market_books[0] market_status = market_book.status market_inplay = market_book.inplay publish_time = market_book.publish_time # Stop the stream if the conditions are met if allow_inplay: if market_status == 'CLOSED': break else: if market_status == 'CLOSED' or market_inplay is True: break insert_in_market_status_table(cursor, publish_time, market_status, market_inplay) for runner in market_book.runners: selection = selections[runner.selection_id] selection_status = runner.status insert_in_selection_status_table(cursor, publish_time, selection, selection_status) for back in runner.ex.available_to_back: insert_in_available_to_back_table(cursor, publish_time, selection, back.price, back.size) for lay in runner.ex.available_to_lay: insert_in_available_to_lay_table(cursor, publish_time, selection, lay.price, lay.size) for volume in runner.ex.traded_volume: insert_in_traded_volume_table(cursor, publish_time, selection, volume.price, volume.size) connection.commit() market_snapshot_no = market_snapshot_no + 1 logger.info("Market snapshot #%s stored.", market_snapshot_no) except KeyboardInterrupt: logger.info("Exiting program (Keyboard interrupt)") break logger.info( "Stopping the stream and logging out from Betfair. This may take a few" " seconds.") stream.stop() try: trading.logout() except APIError: logger.warning("Failed to log out from Betfair: Connection error.") cursor.close() connection.close() logger.info("Compressing the Sqlite file into ZIP file %s", output_zip_file) with zipfile.ZipFile(output_zip_file, 'w', zipfile.ZIP_DEFLATED) as zip_f: zip_f.write(output_sqlite_file, os.path.basename(output_sqlite_file)) os.remove(output_sqlite_file) return output_zip_file
output_queue = queue.Queue() # create stream listener listener = betfairlightweight.StreamListener( output_queue=output_queue, ) # create stream stream = trading.streaming.create_stream( listener=listener, ) # create filters (GB WIN racing) market_filter = streaming_market_filter( event_type_ids=['7'], country_codes=['GB'], market_types=['WIN'], ) market_data_filter = streaming_market_data_filter( fields=['EX_BEST_OFFERS', 'EX_MARKET_DEF'], ladder_levels=3, ) # subscribe streaming_unique_id = stream.subscribe_to_markets( market_filter=market_filter, market_data_filter=market_data_filter, conflate_ms=1000, # send update every 1000ms ) # start stream
self.place_order(market, order) def process_orders(self, market, orders): for order in orders: if order.status == OrderStatus.EXECUTABLE: if order.elapsed_seconds and order.elapsed_seconds > 5: # print(order.bet_id, order.average_price_matched, order.size_matched) if order.size_remaining == 2.00: self.cancel_order(market, order, size_reduction=1.51) # self.update_order(market, order, "PERSIST") # if order.order_type.price == 1.01 and order.size_remaining == 0.49: # self.replace_order(market, order, 1.02) # if order.order_type.price == 1.02: # self.replace_order(market, order, 1.03) # if order.order_type.price == 1.03: # self.replace_order(market, order, 1.05) pass trading = betfairlightweight.APIClient("username") client = clients.BetfairClient(trading) framework = Flumine(client=client) strategy = ExampleStrategy( market_filter=streaming_market_filter(market_ids=["1.170378175"]), ) framework.add_strategy(strategy) framework.run()
trading = bf_utils.api_login() # grab a horse race test_market = bf_utils.grab_market_id() # create queue output_queue = queue.Queue() # create stream listener listener = betfairlightweight.StreamListener(output_queue=output_queue) # create stream stream = trading.streaming.create_stream(listener=listener) # create filters (GB WIN racing) market_filter = streaming_market_filter(market_ids=[test_market]) market_data_filter = streaming_market_data_filter( fields=["EX_MARKET_DEF", "EX_BEST_OFFERS"], ladder_levels=3) # subscribe streaming_unique_id = stream.subscribe_to_markets( market_filter=market_filter, market_data_filter=market_data_filter, conflate_ms=1000, # send update every 1000ms ) # start stream in a new thread (in production would need err handling) t = threading.Thread(target=stream.start, daemon=True) t.start() # check for updates in output queue
runner_name = runner_names[runner.selection_id] best_back_price = utils.get_price(runner.ex.available_to_back, 0) # betconnect selection = selections_lookup[runner_name] max_price = selection.max_price diff = (1 / max_price) - (1 / best_back_price) print(runner_name, best_back_price, max_price, round(diff, 3)) framework = Flumine() # add clients betfair_client = clients.BetfairClient( betfairlightweight.APIClient("username")) framework.add_client(betfair_client) betconnect_client = clients.BetConnectClient( betconnect.APIClient("username", "password", "apiKey", "ppURL")) framework.add_client(betconnect_client) strategy = ExampleStrategy( market_filter=streaming_market_filter(market_ids=["1.196548740"]), streaming_timeout=2, ) framework.add_strategy(strategy) framework.run()
async def send_subscription_message( self, market_ids: list = None, betting_types: list = None, event_type_ids: list = None, event_ids: list = None, turn_in_play_enabled: bool = None, market_types: list = None, venues: list = None, country_codes: list = None, race_types: list = None, initial_clk: str = None, clk: str = None, conflate_ms: int = None, heartbeat_ms: int = None, segmentation_enabled: bool = True, subscribe_book_updates=True, subscribe_trade_updates=True, subscribe_market_definitions=True, ): """ See `betfairlightweight.filters.streaming_market_filter` for full docstring :param market_ids: :param betting_types: :param event_type_ids: :param event_ids: :param turn_in_play_enabled: :param market_types: :param venues: :param country_codes: :param race_types: :param subscribe_book_updates: Subscribe to market orderbook events :param subscribe_trade_updates: Subscribe to market trade events :return: """ filters = ( market_ids, betting_types, event_type_ids, event_ids, turn_in_play_enabled, market_types, venues, country_codes, race_types, ) assert any(filters), "Must pass at least one filter" assert any( (subscribe_book_updates, subscribe_trade_updates) ), "Must subscribe to either book updates or trades" if market_ids is not None: # TODO - Log a warning about inefficiencies of specific market ids - Won't receive any updates for new # markets that fit criteria like when using event type / market type etc # logging.warning() pass market_filter = streaming_market_filter( market_ids=market_ids, betting_types=betting_types, event_type_ids=event_type_ids, event_ids=event_ids, turn_in_play_enabled=turn_in_play_enabled, market_types=market_types, venues=venues, country_codes=country_codes, race_types=race_types, ) data_fields = [] if subscribe_book_updates: data_fields.append("EX_ALL_OFFERS") if subscribe_trade_updates: data_fields.append("EX_TRADED") if subscribe_market_definitions: data_fields.append("EX_MARKET_DEF") market_data_filter = streaming_market_data_filter( fields=data_fields, ) message = { "op": "marketSubscription", "id": self.unique_id, "marketFilter": market_filter, "marketDataFilter": market_data_filter, "initialClk": initial_clk, "clk": clk, "conflateMs": conflate_ms, "heartbeatMs": heartbeat_ms, "segmentationEnabled": segmentation_enabled, } await self.send(raw=message)
# create trading instance (app key must be activated for streaming) username = os.environ.get('username') trading = betfairlightweight.APIClient(username) trading.login() # create queue output_queue = queue.Queue() # create stream listener listener = betfairlightweight.StreamListener(output_queue=output_queue, ) # create stream stream = trading.streaming.create_stream(listener=listener, ) # create filters (GB WIN racing) market_filter = streaming_market_filter(market_ids=['156436827']) market_data_filter = streaming_market_data_filter() # subscribe streaming_unique_id = stream.subscribe_to_markets( market_filter=market_filter, market_data_filter=market_data_filter, conflate_ms=1000, # send update every 1000ms ) # start stream stream.start(_async=True) """ Data can also be accessed by using the snap function in the listener, e.g: market_books = listener.snap(
score = market.context["score"] print( score.match_status, score.current_game, score.current_set, score.current_point, score.score.home.score, score.score.away.score, ) trading = betfairlightweight.APIClient("username") client = clients.BetfairClient(trading) framework = Flumine(client=client) strategy = ExampleStrategy( market_filter=streaming_market_filter(market_ids=["1.172415939"]), ) framework.add_strategy(strategy) framework.add_worker( BackgroundWorker( framework, poll_in_play_service, func_kwargs={"event_type_id": "2"}, interval=30, start_delay=4, )) framework.run()
def data_collection_pipeline() -> str: """Pipeline to collect Betfair odds market ladder streaming data. Returns: Path of the output CSV file. """ logging.basicConfig( format='%(levelname)-8s | %(asctime)s | %(name)s: %(message)s', level=logging.INFO, # stream=sys.stdout ) market_id, output_dir, conflate_ms = parse_command_line_args() trading = bfl.APIClient(username=username, password=password, app_key=app_key, cert_files=[cert_file, cert_key_file]) logger.info("Logging in to Betfair") trading.login() # Event and market information event_type, event, competition = get_event_info(trading, market_id) market_name, market_start_time, selections = get_market_info( trading, market_id) # Output file path output_file_name = get_output_file_name(event_type, event, competition, market_name, market_start_time) output_file = os.path.join(output_dir, output_file_name) # Market stream logger.info("Initialising output queue") output_queue = queue.Queue() logger.info("Initialising Betfair stream listener") listener = bfl.StreamListener(output_queue) logger.info("Creating the Betfair market stream") stream = trading.streaming.create_stream(listener=listener) logger.info("Setting the market filter to market_id=%s", market_id) market_filter = streaming_market_filter(market_ids=[market_id]) logger.info("Initialising streaming market data filter") market_data_filter = streaming_market_data_filter( fields=['EX_MARKET_DEF', 'EX_ALL_OFFERS', 'EX_TRADED'], ) logger.info("Subscribing to the market") stream.subscribe_to_markets(market_filter=market_filter, market_data_filter=market_data_filter, conflate_ms=conflate_ms) logger.info("Starting the stream") stream.start(async_=True) logger.info(f"Saving data in file {output_file}") with open(output_file, 'w') as f: f_csv = csv.writer(f) csv_header = ['selection', 'time', 'price', 'size', 'side'] f_csv.writerow(csv_header) market_snapshot_no = 0 while True: try: market_books = output_queue.get() market_book = market_books[0] publish_time = market_book.publish_time rows = [] for runner in market_book.runners: selection_id = runner.selection_id for back in runner.ex.available_to_back: rows.append((selections[selection_id], publish_time, back.price, back.size, 'back')) for lay in runner.ex.available_to_lay: rows.append((selections[selection_id], publish_time, lay.price, lay.size, 'lay')) f_csv.writerows(rows) market_snapshot_no = market_snapshot_no + 1 logger.info("Market snapshot #%s stored.", market_snapshot_no) except KeyboardInterrupt: logger.info("Exiting program (Keyboard interrupt)") break logger.info("Stopping the stream and logging out from Betfair.") stream.stop() trading.logout() return output_file
def test_streaming_market_filter(self): response = streaming_market_filter() assert response == {} response = streaming_market_filter(market_ids=[1, 2]) assert response == {"marketIds": [1, 2]}
log_handler = logging.StreamHandler() formatter = jsonlogger.JsonFormatter(custom_format) formatter.converter = time.gmtime log_handler.setFormatter(formatter) logger.addHandler(log_handler) logger.setLevel(logging.INFO) class ExampleStrategy(BaseStrategy): def process_sports_data(self, market, sports_data) -> None: # called on each update from sports-data-stream print(market, sports_data) trading = betfairlightweight.APIClient("username") client = clients.BetfairClient(trading) framework = Flumine(client) strategy = ExampleStrategy( market_filter=streaming_market_filter( event_type_ids=["4"], market_types=["MATCH_ODDS"] ), sports_data_filter=[ "cricketSubscription" ], # "cricketSubscription" and/or "raceSubscription" ) framework.add_strategy(strategy) framework.run()