def test_historical_stream(self):
trading = betfairlightweight.APIClient("username",
"password",
app_key="appKey")
stream = trading.streaming.create_historical_stream(
file_path="tests/resources/historicaldata/RACE-1.140075353",
listener=StreamListener(),
operation="raceSubscription",
)
stream.start()
for cache in stream.listener.stream._caches.values():
cache.create_resource(1, False)
assert stream.listener.stream_type == "raceSubscription"
assert stream.listener.stream_unique_id == 0
assert stream.listener.stream._updates_processed == 4
assert len(stream.listener.stream._caches) == 2
market = stream.listener.stream._caches.get("1.1234567")
assert len(market.rrc) == 2
market = stream.listener.stream._caches.get("1.173853449")
assert len(market.rrc) == 4
assert stream._running is False
def __init__(self, username, password, app_key):
self.api = betfairlightweight.APIClient(username,
password,
app_key=app_key)
self.api.login_interactive()
self.leagues_list, self.last_since_id = {}, {}
self.init_data()
self.logger = init_logger()
self.grouped_books = []
self.football_filter = filters.market_filter(
event_type_ids=[1], # filter on just soccer
competition_ids=self.FOOTBALL_COMPETITION_IDS,
market_type_codes=[
"OVER_UNDER_25", "MATCH_ODDS", "ASIAN_HANDICAP"
])
self.esports_filter = filters.market_filter(
event_type_ids=[27454571], # filter on esports
competition_ids=self.ESPORT_COMPETITION_IDS,
market_type_codes=["MATCH_ODDS"
], # filter on just odds market types
)
self.default_market_projection = [
#"RUNNER_DESCRIPTION",
"COMPETITION",
"EVENT",
"EVENT_TYPE",
#"MARKET_DESCRIPTION",
"MARKET_START_TIME",
]
def betfair_client():
mock.patch("betfairlightweight.endpoints.login.Login.__call__")
return betfairlightweight.APIClient(
username="******",
password="******",
app_key="app_key",
certs="cert_location",
)
def betfair_client():
mock.patch("betfairlightweight.endpoints.login.Login.__call__")
return betfairlightweight.APIClient( # noqa (S106 Possible hardcoded password: '******')
username="******",
password="******",
app_key="app_key",
certs="cert_location",
)
def conectaNaBetFair():
# create trading instance
trading = betfairlightweight.APIClient(usuarioAPI,
senhaAPI,
app_key=APIKey,
certs='d:/python/codes/botfair/')
# login
trading.login()
return trading
def conectaNaBetFair():
# create trading instance
import pathlib
trading = betfairlightweight.APIClient(usuarioAPI,
senhaAPI,
app_key=APIKey,
certs=str(
pathlib.Path().absolute()))
# login
trading.login()
return trading
def __init__(self):
logger.info("Initialising Betfair client.")
self._client = betfairlightweight.APIClient(
username=BetfairConfig.USERNAME,
password=BetfairConfig.PASSWORD,
app_key=BetfairConfig.APP_KEY,
cert_files=[BetfairConfig.CERT_FILE, BetfairConfig.CERT_KEY_FILE])
logger.info("Logging in to Betfair.")
self._client.login()
self.stream: Union[
None,
betfairlightweight.streaming.betfairstream.BetfairStream] = None
def login():
file_path = os.path.dirname(os.path.abspath(__file__))
certs_path = f"{file_path}/../certs"
# create trading instance
keys = json.load(open(f"{file_path}/../keys.json"))
trading = betfairlightweight.APIClient(username=keys['betfair-username'],
password=keys['betfair-password'],
app_key=keys['betfair-key-delay'],
certs=certs_path,
locale='italy')
# login
trading.login()
return trading
def trading_instance():
'''
Connect to the api.
Requires 'api_logins.json' file and certs folder in current directory.
'''
with open('api_logins.json') as f:
login_dict = json.load(f)
trading = betfairlightweight.APIClient(username=login_dict['my_username'],
password=login_dict['my_password'],
app_key=login_dict['my_app_key'],
certs=certs_path)
return(trading)
async def test_client_connect(live_logger):
betfair_client = betfairlightweight.APIClient(
username=os.environ["BETFAIR_USERNAME"],
password=os.environ["BETFAIR_PW"],
app_key=os.environ["BETFAIR_APP_KEY"],
certs=os.environ["BETFAIR_CERT_DIR"],
)
# TODO - mock login won't let you login - need to comment out in conftest.py to run
betfair_client.login()
socket = BetfairMarketStreamClient(client=betfair_client,
logger=live_logger,
message_handler=print)
await socket.connect()
await socket.send_subscription_message(market_ids=["1.180634014"])
await asyncio.sleep(15)
def test_historical_generator_stream(self):
# assert that data is processed correctly (regression testing)
trading = betfairlightweight.APIClient("username",
"password",
app_key="appKey")
stream = trading.streaming.create_historical_generator_stream(
file_path="tests/resources/historicaldata/BASIC-1.132153978",
listener=StreamListener(lightweight=True),
)
gen = stream.get_generator()
data = [i[0] for i in gen()]
with open(
"tests/resources/historicaldata/BASIC-1.132153978-processed.json",
"r") as f:
expected_data = load(f)
assert expected_data == data
def test_historical_stream(self):
trading = betfairlightweight.APIClient('username', 'password', app_key='appKey')
stream = trading.streaming.create_historical_stream(
directory='tests/resources/historicaldata/BASIC-1.132153978',
listener=StreamListener()
)
stream.start(_async=False)
assert stream.listener.stream_type == 'marketSubscription'
assert stream.listener.stream_unique_id == 'HISTORICAL'
assert stream.listener.clk == '3522512789'
assert stream.listener.stream._updates_processed == 480
assert len(stream.listener.stream._caches) == 1
market = stream.listener.stream._caches.get('1.132153978')
assert len(market.runners) == 14
assert stream._running is False
async def test_betfair_data_client(betfair_data_client, data_engine):
""" Local test only, ensure we can connect to betfair and receive some market data """
betfair_client = betfairlightweight.APIClient(
username=os.environ["BETFAIR_USERNAME"],
password=os.environ["BETFAIR_PW"],
app_key=os.environ["BETFAIR_APP_KEY"],
certs=os.environ["BETFAIR_CERT_DIR"],
)
betfair_client.login()
def printer(x):
print(x)
# TODO - mock betfairlightweight.APIClient.login won't let this pass, need to comment out to run
socket = BetfairMarketStreamClient(client=betfair_client,
message_handler=printer)
await socket.connect()
await socket.send_subscription_message(market_ids=["1.180634014"])
await socket.start()
def test_historical_stream(self):
trading = betfairlightweight.APIClient("username",
"password",
app_key="appKey")
stream = trading.streaming.create_historical_stream(
file_path="tests/resources/historicaldata/BASIC-1.132153978",
listener=StreamListener(),
)
stream.start()
assert stream.listener.stream_type == "marketSubscription"
assert stream.listener.stream_unique_id == 0
assert stream.listener.clk == "3522512789"
assert stream.listener.stream._updates_processed == 480
assert len(stream.listener.stream._caches) == 1
market = stream.listener.stream._caches.get("1.132153978")
assert len(market.runners) == 14
assert stream._running is False
def __init__(self,
username: str,
password: str,
app_key: str,
cert_path: str,
sport: str = None,
plan: str = None,
from_date: datetime = None,
to_date: datetime = None):
"""
Creates an instance to allow for interaction with the betfairlightweight API.
Optional arguments are required for all functions other than getting available data.
Once known parameters are known these must be passed to use other functions.
:param username: Betfair username
:param password: Betfair password
:param app_key: Application key for Betfair account
:param cert_path: Directory to certificates
::param sport: Betfair sport
:param plan: Betfair plan
:param from_date: Datetime of earliest date to collect
:param to_date: Datetime of latest date to collect
"""
self.username = username
self.cert_path = cert_path
self.sport = sport
self.plan = plan
self.from_date = from_date
self.to_date = to_date
self.trading = betfairlightweight.APIClient(username=username,
password=password,
app_key=app_key,
certs=cert_path)
self.trading.login()
self.available_data = self.trading.historic.get_my_data()
def test_historical_stream(self):
trading = betfairlightweight.APIClient("username",
"password",
app_key="appKey")
stream = trading.streaming.create_historical_stream(
file_path="tests/resources/historicaldata/CRICKET-1.179676557",
listener=StreamListener(),
operation="cricketSubscription",
)
stream.start()
for cache in stream.listener.stream._caches.values():
cache.create_resource(1, False)
assert stream.listener.stream_type == "cricketSubscription"
assert stream.listener.stream_unique_id == 0
assert stream.listener.stream._updates_processed == 3
assert len(stream.listener.stream._caches) == 1
market = stream.listener.stream._caches.get("1.179676557")
assert market is not None
assert stream._running is False
import betfairlightweight
trading = betfairlightweight.APIClient("*****@*****.**","Passwordshouldgohere", app_key="7Y1J88OHOt5f96hV")
trading.login_interactive()
file_list = trading.historic.get_file_list(
"Soccer", "Basic Plan", 15, 12, 2019, 19, 12, 2019, market_types_collection=["MATCH_ODDS"],
countries_collection=['GB'], file_type_collection=['M'])
for file in file_list:
print(file)
download = trading.historic.download_file(file_path=file)
print(download)
# at the start of the program, get all games for betfair that day and using the list of today's games from smarkets, discard those not in betfair
# create a thread to check all betfair live game odds every second
# create a separate thread for each live smarkets games using selenium and check each every second
# with each check of the odds for a game, record the results to a function that can be accessed by both the smarkets and betfair threads
# when an arb is found by comparing the odds recorded in the above function, notify
# stop getting data for relevant game when betfair market for the match odds returns an error, as game has ended
# don't forget 5% and 2% comission
# set up betfair
import betfairlightweight
from betfairlightweight import filters
print('setting up betfair api')
trading = betfairlightweight.APIClient(username='',
password='',
app_key='',
certs=r'')
trading.login()
event_types = trading.betting.list_event_types()
football_event_filter = betfairlightweight.filters.market_filter(
event_type_ids=[1],
in_play_only=True,
market_start_time={
'from': '2020-07-30T00:00:00Z',
'to': '2020-07-31T00:00:00Z'
})
football_events_raw = trading.betting.list_events(filter=football_event_filter)
football_events = [(event_object.event.name, event_object.event.id)
for event_object in football_events_raw]
print(len(football_events))
my_path = '/home/dump/in_database'
my_path = '/Users/vincent.mele/testmarkets'
psql_dsn = {
'host': 'localhost,s3.vpn.mele.law',
'port': '5432',
'user': '******',
'dbname': 'betfair'
}
db = db.DB(psql_dsn=psql_dsn)
# setup logging
logging.basicConfig(level=logging.INFO)
# create trading instance (don't need username/password)
trading = betfairlightweight.APIClient("username", "password", app_key='')
# create listener
listener = StreamListener(max_latency=None)
def snapshot(_db, _market_book, _seconds_to_start):
_raceid = int(_market_book.market_id.replace('1.1', '11'))
_milliseconds_to_start = _seconds_to_start * 1000
_event_type = int(_market_book.market_definition.event_type_id)
_country_code = _market_book.market_definition.country_code
try:
_d = market_book.json().decode('utf-8')
except:
_d = market_book.json()
def betfair_login():
"""Log into the Betfair Exchange API using credentials stored in configuration.py"""
trading = betfairlightweight.APIClient(
username=BFEX_USERNAME, password=BFEX_PASSWORD, app_key=BFEX_APP_KEY, certs=BFEX_CERTS_DIR)
return trading
import re
import pandas as pd
import numpy as np
from bz2 import BZ2File
#### logging in
project_dir = Path.cwd().parents[1]
logins_dir = project_dir / 'api_logins.json'
with open(logins_dir) as f:
login_dict = json.load(f)
trading = betfairlightweight.APIClient(username=login_dict['my_username'],
password=login_dict['my_password'],
app_key=login_dict['my_app_key'],
certs=login_dict['certs_path'])
trading.login()
### retreive api list
data_dicts = trading.historic.get_my_data()
adv_range = [d['forDate'] for d in data_dicts if d['plan'] == 'Advanced Plan']
adv_min_date = datetime.datetime.strptime(min(adv_range), '%Y-%m-%dT%H:%M:%S')
def last_day_of_month(any_day):
next_month = any_day.replace(day=28) + datetime.timedelta(days=4)
return next_month - datetime.timedelta(days=next_month.day)
while True:
try:
# Wait until 8am
today = datetime.date.today()
wait_until(
datetime.datetime(today.year, today.month, today.day, 8, 0, 0))
# Create DB session
db_session = Session()
print('DB session created')
# Logon to Betfair
betfair_api = bfl.APIClient(
secrets['username'],
secrets['password'],
app_key=secrets['betangel_app_key'],
cert_files=['./certs/client-2048.crt', './certs/client-2048.key'],
lightweight=False)
betfair_api.login()
print('Logged on to Betfair')
# Log today's markets
log_markets(db_session, betfair_api, today)
# Logout from Betfair
betfair_api.logout()
print('Logged off from Betfair')
betfair_api = None
## Close DB session
db_session.close()
from betfairlightweight.resources.bettingresources import (PriceSize,
MarketBook)
file_output = "output_bflw.csv"
market_paths = [
"data/2021_10_OctRacingAUPro.tar",
"data/2021_11_NovRacingAUPro.tar",
"data/2021_12_DecRacingAUPro.tar",
]
# setup logging
logging.basicConfig(level=logging.FATAL)
# create trading instance (don't need username/password)
trading = betfairlightweight.APIClient("username", "password", "appkey")
# create listener
listener = betfairlightweight.StreamListener(
max_latency=None, # ignore latency errors
output_queue=None, # use generator rather than a queue (faster)
lightweight=False, # lightweight mode is faster
update_clk=
False, # do not update clk on updates (not required when backtesting)
cumulative_runner_tv=True,
calculate_market_tv=True)
# loading from tar and extracting files
def load_markets(file_paths: List[str]):
for file_path in file_paths:
== selection["selection_id"]):
trade = Trade(
market_id=market_book.market_id,
selection_id=runner.selection_id,
handicap=runner.handicap,
strategy=self,
)
order = trade.create_order(
side=selection["side"],
order_type=LimitOrder(price=1.01,
size=selection["liability"]),
)
market.place_order(order)
trading = betfairlightweight.APIClient("username")
client = clients.BetfairClient(trading)
framework = Flumine(client=client)
strategy = SingleStrategy(
name="back_strat_42",
market_filter=streaming_market_filter(
event_type_ids=["7"],
country_codes=["GB", "IE"],
market_types=["WIN"],
),
market_data_filter=streaming_market_data_filter(
fields=[
"EX_BEST_OFFERS",
"EX_LTP",
def data_collection_pipeline() -> str:
"""Pipeline to collect Betfair odds market ladder streaming data.
Returns:
Path of the output CSV file.
"""
logging.basicConfig(
format='%(levelname)-8s | %(asctime)s | %(name)s: %(message)s',
level=logging.INFO)
market_id, output_dir, conflate_ms, no_virtual_bets, allow_inplay,\
mins_before_start = parse_command_line_args()
trading = bfl.APIClient(username=username,
password=password,
app_key=app_key,
cert_files=[cert_file, cert_key_file])
logger.info("Logging in to Betfair")
trading.login()
# Event and market information
event_type, event, competition = get_event_info(trading, market_id)
market_name, market_start_time, selections = get_market_info(
trading, market_id)
# Wait to stream until a certain amount of minutes before the start
if mins_before_start is not None:
logger.info(
"Logging off from Betfair and waiting until %s minutes before the "
"start of the event. Press Ctrl+C to quit.", mins_before_start)
trading.logout()
now = datetime.utcnow()
try:
while market_start_time - now >= \
timedelta(minutes=mins_before_start):
time.sleep(1)
now = datetime.utcnow()
except KeyboardInterrupt:
logger.info("Exiting program (Keyboard interrupt)")
exit(0)
logger.info("Logging in to Betfair again.")
trading.login()
# Output file path
output_file_name = get_output_file_name(event_type, event, competition,
market_name, market_start_time)
output_sqlite_file = os.path.join(output_dir, output_file_name + '.db')
output_zip_file = os.path.join(output_dir, output_file_name + '.zip')
# Market stream
logger.info("Initialising output queue")
output_queue = queue.Queue()
logger.info("Initialising Betfair stream listener")
listener = bfl.StreamListener(output_queue)
logger.info("Creating the Betfair market stream")
stream = trading.streaming.create_stream(listener=listener)
logger.info("Setting the market filter to market_id=%s", market_id)
market_filter = streaming_market_filter(market_ids=[market_id])
logger.info("Initialising streaming market data filter")
if no_virtual_bets:
market_data_fields = ['EX_MARKET_DEF', 'EX_ALL_OFFERS', 'EX_TRADED']
else:
market_data_fields = [
'EX_MARKET_DEF', 'EX_BEST_OFFERS_DISP', 'EX_TRADED'
]
market_data_filter = streaming_market_data_filter(
fields=market_data_fields, )
logger.info("Subscribing to the market")
stream.subscribe_to_markets(market_filter=market_filter,
market_data_filter=market_data_filter,
conflate_ms=conflate_ms)
logger.info("Starting the stream")
stream.start(async_=True)
logger.info(f"Saving data in file {output_sqlite_file}")
connection, cursor = create_sqlite_database(output_sqlite_file)
market_snapshot_no = 0
while True:
try:
market_books = output_queue.get()
market_book = market_books[0]
market_status = market_book.status
market_inplay = market_book.inplay
publish_time = market_book.publish_time
# Stop the stream if the conditions are met
if allow_inplay:
if market_status == 'CLOSED':
break
else:
if market_status == 'CLOSED' or market_inplay is True:
break
insert_in_market_status_table(cursor, publish_time, market_status,
market_inplay)
for runner in market_book.runners:
selection = selections[runner.selection_id]
selection_status = runner.status
insert_in_selection_status_table(cursor, publish_time,
selection, selection_status)
for back in runner.ex.available_to_back:
insert_in_available_to_back_table(cursor, publish_time,
selection, back.price,
back.size)
for lay in runner.ex.available_to_lay:
insert_in_available_to_lay_table(cursor, publish_time,
selection, lay.price,
lay.size)
for volume in runner.ex.traded_volume:
insert_in_traded_volume_table(cursor, publish_time,
selection, volume.price,
volume.size)
connection.commit()
market_snapshot_no = market_snapshot_no + 1
logger.info("Market snapshot #%s stored.", market_snapshot_no)
except KeyboardInterrupt:
logger.info("Exiting program (Keyboard interrupt)")
break
logger.info(
"Stopping the stream and logging out from Betfair. This may take a few"
" seconds.")
stream.stop()
try:
trading.logout()
except APIError:
logger.warning("Failed to log out from Betfair: Connection error.")
cursor.close()
connection.close()
logger.info("Compressing the Sqlite file into ZIP file %s",
output_zip_file)
with zipfile.ZipFile(output_zip_file, 'w', zipfile.ZIP_DEFLATED) as zip_f:
zip_f.write(output_sqlite_file, os.path.basename(output_sqlite_file))
os.remove(output_sqlite_file)
return output_zip_file
from betfairlightweight import filters
from secrets import (
address,
password,
certs,
country,
sport,
app_key,
teams,
rating_teams,
)
from ast import literal_eval
# create trading instance
trading = betfairlightweight.APIClient(address,
password,
app_key=app_key,
certs=certs)
# login
trading.login()
# update for test
def place_order():
file = open("dict.txt")
x = file.read()
y = literal_eval(x)
file = open("dict_old.txt")
a = file.read()
b = literal_eval(a)
def __init__(self):
self.trading = betfairlightweight.APIClient(**_get_credentials())
self.trading.login()
self.matches_df = None
self.data = None
# This tutorial is based off the betfairlightweight stream tutorials
# which are available at
# https://github.com/liampauling/betfair/blob/master/examples/examplestreaming.py
# Change this certs path to wherever you're storing your certificates
certs_path = "your_certs_path"
# Change these login details to your own
username = "******"
pw = "your_password"
app_key = "your_app_key"
# Log in - if you have set up your certs use this
print("Logging in")
trading = betfairlightweight.APIClient(username,
pw,
app_key=app_key,
certs=certs_path)
trading.login()
# Log in - if you haven't set up your certs use this
# trading = betfairlightweight.APIClient(username, pw, app_key=app_key)
# trading.login_interactive()
# create queue
output_queue = queue.Queue()
# create stream listener
print('Creating listener')
listener = betfairlightweight.StreamListener(output_queue=output_queue, )
# create stream
import logging
import betfairlightweight
# setup logging
logging.basicConfig(
level=logging.INFO) # change to DEBUG to see log all updates
# create trading instance
trading = betfairlightweight.APIClient('username', 'password')
# update
market_id = '1.133559518'
# race card login
trading.race_card.login()
# race card request (provide list / returns list)
race_cards = trading.race_card.get_race_card(market_ids=[market_id])
print(race_cards)
for race_card in race_cards:
print(
race_card,
race_card.prize,
race_card.timeform_123_text,
) # view resources or debug to see all values available
for runner in race_card.runners:
print(
runner.name,
start = timer()
# Login to API Client
# =================
# Change this certs path to wherever you're storing your certificates
certs_path = "/Users/17macm/Other/Programming/certs"
# Change these login details to your own
my_username = "******"
my_password = "******"
my_app_key = "LFpPp46vq2zDrYmR"
trading = betfairlightweight.APIClient(username=my_username,
password=my_password,
app_key=my_app_key,
certs=certs_path)
trading.login()
"""
# Grab all event type ids. This will return a list which we will iterate over to print out the id and the name of the sport
event_types = trading.betting.list_event_types()
sport_ids = pd.DataFrame({
'Sport': [event_type_object.event_type.name for event_type_object in event_types],
'ID': [event_type_object.event_type.id for event_type_object in event_types]
}).set_index('Sport').sort_index()
sport_ids
"""
