def query_neural_network_for_bets(X):
""" input X has to be a np array of shape [number_of_requested_predictions, elements]
with ts starting at t-000.
Returns betting recommendation in the form of bet,lay,bet,lay... as booleans in the same order as X
Return shape for 2 bets and 44 horses would be (2,88)
"""
config = get_config()
nn_class = getattr(sys.modules[__name__],
config.get('Betting', 'model_class'))
n = nn_class()
n.load_model()
m = MongoManager()
d = {}
d['input'] = X.tolist()
d['timestamp'] = datetime.now()
predicted = (n.predict(X))
d['output'] = predicted.tolist()
log.info("Mean predicted profit across all bets: {}".format(
np.mean(predicted)))
if np.isnan(np.mean(predicted)):
log.error("Tensorflow output: {}".format(predicted))
log.error("Terminating thread")
sys.exit()
place_bet = predicted
return place_bet, n.model_name
def calculate_payoffs(self):
self.df = self.df.fillna(0)
config = get_config()
fees = config.getfloat("Betting", "fees")
stake = 1 # hard coded for training for comparison purposes
""" Creates X and Y in assigning the payoff to Y by removing the winner column from original df """
payoff_back = np.where(self.df['winner'].values,
(stake * self.df['LTP'].values - stake) * (1 - fees), # back winner
-np.ones((self.df['LTP'].values.shape)) * stake) # back loser
payoff_back = np.where(self.df['winner'] != -1, payoff_back, 0) # replace na with 0 payoff
# payoff when lay: -odds when win, 1 when loss
payoff_lay = np.where(self.df['winner'].values,
-(stake * self.df['LTP'].values - stake), # lay winner
stake * np.ones((self.df['LTP'].values.shape)) * (1 - fees)) # lay loser
payoff_lay = np.where(self.df['winner'] != -1, payoff_lay, 0) # replace na with 0 payoff
# keep the lay odds to analyse the potential risk on the lays we bet on
winning_lay_risk = np.minimum((self.df['LTP'].values - 1) * (-1), np.zeros(payoff_lay.shape))
self.df['back'] = payoff_back
self.df['lay'] = payoff_lay
self.df['lay_risk'] = winning_lay_risk
def get_score(self):
config = get_config()
self.back_pnl = [0]
self.lay_pnl = [0]
fees = config.getfloat("Betting", "fees")
stake = config.getfloat("Betting", "Stake")
won = np.nan
loser = np.nan
LTP = self.ltp
if won and self.back: # back on winner
change = (LTP * stake - stake) * (1 - fees)
self.back_pnl.append(change)
if won and self.lay: # lay on winner
change = -(LTP * stake - stake)
self.lay_pnl.append(change)
if loser and self.back: # back on loser
change = -stake # pylint: disable=E1130
self.back_pnl.append(change)
if loser and self.lay: # lay on loser
change = stake * (1 - fees)
self.lay_pnl.append(change)
total_profit = sum(self.lay_pnl) + sum(self.back_pnl)
return total_profit
def setUp(cls):
container = Container()
cls.client = container.client
config = get_config()
prodkey = config.get("Betfair", "prod_app")
certfile = config.get("Betfair", "cert_file")
cls.client = Betfair(prodkey, certfile)
cls.client.session_token = 'FakeSession'
def do_login(self):
config = get_config()
username = config.get("Matchbook", "username")
password = config.get("Matchbook", "password")
self.api = APIClient(username, password)
# try:
self.api.login()
sport_ids = self.api.reference_data.get_sports()
self.horse_racing = list(
filter(lambda x: x['name'] == 'Horse Racing', sport_ids))[0]['id']
def check_for_available_race_and_bet(args):
from horse_racing.utils.tools import get_config
config = get_config()
min_before_start = config.get("Betting", "min_before_start_initial_bet")
reference_price_for_initial_bet = config.get(
"Betting", "reference_price_for_initial_bet")
countrycodes = config.get('Betting', 'bet_countries').split()
races_to_bet, selection_ids, market_ids, event_ids, race_start_times = \
find_race_to_bet(min_before_start=int(min_before_start), countrycodes=countrycodes)
wait_for_start = int(min_before_start) == 0
if races_to_bet is not None:
b = BetLogic()
b.update_prices_and_place_bet(args, races_to_bet, selection_ids,
market_ids, event_ids, race_start_times,
wait_for_start,
reference_price_for_initial_bet)
def update_placed_orders(self, armed=False):
config = get_config()
m = MongoManager()
open_orders = self.container.get_open_orders()
use_level = config.get("Betting", "use_level").split(',')
# minutes_to_start
if len(open_orders) > 0:
log.info("Currently open order: {}".format(len(open_orders)))
for market, orders in open_orders.items():
log.info("Updating order for market: {}".format(market))
new_prices = self.container.get_single_market(market)
updated_prices = []
updated_betids = []
for o in orders:
selection_id = o.selection_id
secs_to_start = m.get_secs_to_start(market)
log.info("Seconds to start: {}".format(secs_to_start))
if not secs_to_start or secs_to_start < \
-config.getint("Betting", "update_until_secs_after_start"): # pylint: disable=E1130
log.info("Ignoring order\n----------------")
continue
log.info("Selection ID: {}".format(selection_id))
ltp_price = 0
lays = [0, 0, 0]
backs = [0, 0, 0]
for r in new_prices[0].runners:
if r.selection_id != selection_id:
continue
backs = [
r.ex.available_to_back[x].price
for x in range(len(r.ex.available_to_back))
]
lays = [
r.ex.available_to_lay[x].price
for x in range(len(r.ex.available_to_lay))
]
ltp_price = r.last_price_traded
log.info("New backs: {}".format(backs))
log.info("New lays: {}".format(lays))
min_to_start = math.floor(secs_to_start / 60)
log.info("Minutes until start: {}".format(min_to_start))
try:
level_at_minutes_to_start = int(use_level[min_to_start])
log.info(
"Using Level: {}".format(level_at_minutes_to_start))
except IndexError:
log.info(
"use_level not set for {} minutes to start. Aborting".
format(min_to_start))
return
level_at_minutes_to_start = min(level_at_minutes_to_start,
len(backs) + 1)
log.info(
"Final level to use: {}".format(level_at_minutes_to_start))
new_price = ltp_price if level_at_minutes_to_start == 0 else backs[
level_at_minutes_to_start - 1]
log.info("Updating to new price: {}".format(new_price))
new_price = lay_saftey_check(selection_id=o.selection_id,
last_price=ltp_price,
current_lay_price=new_price,
best_back=backs[0])
new_price = price_adjustment(new_price)
updated_betids.append(o.bet_id)
updated_prices.append(new_price)
if armed:
m.update_order_price(o.market_id, o.selection_id, o.bet_id,
min_to_start, new_price)
if len(updated_betids) > 0:
log.info("Updated bet_ids: {}".format(updated_betids))
log.info("Updated prices: {}".format(updated_prices))
if armed:
self.container.replace_orders(market, updated_betids,
updated_prices)
import logging
import socket
import sys
from datetime import timedelta, datetime
import matplotlib
from scipy.stats import ttest_ind
from tqdm import tqdm
from horse_racing.utils.tools import get_config, chunk
config = get_config()
if socket.gethostname() == config.get("Servers", "prod"):
matplotlib.use(
'Agg') # to work with linux, but will prevent showing plot in windows
from horse_racing.utils.mongo_manager import MongoManager
import pandas as pd
import os
import matplotlib.pyplot as plt
import numpy as np
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import smtplib
from horse_racing.betfair_manager.engine import Container
log = logging.getLogger(__name__)
def propagate_race_results():
log.info("Propagating race results")
def collect_prices(collection_name='price_scrape', single_marketid=False):
from horse_racing.betfair_manager.engine import Container as BFContainer
from horse_racing.matchbook_manager.engine import Container as MBContainer
try:
bfm = BFContainer()
except:
log.error("Unable to log into betfair")
if not single_marketid:
try:
mbm = MBContainer()
except:
log.error("Unable to log into matchbook")
if single_marketid:
events, markets = bfm.get_single_race(single_marketid)
else:
events, markets = bfm.get_all_races()
try:
prices = bfm.update_markets(events, markets)
except AttributeError:
log.info("No markets to scrape")
return
if not single_marketid:
try:
mb_prices = mbm.get_races()
except:
mb_prices = {}
log.error("failed to get mb prices")
else:
mb_prices = {}
event_mapping = {}
bf_selection_name_mapping = {}
for e, ms in markets.items():
for m in ms:
mb_market = None
try:
mb_market = mb_prices[(
m[1].event.venue,
m[1].market_start_time.strftime("%Y-%m-%dT%H:%M:00.000Z"))]
except KeyError:
log.info("Didn't find matchbook market for {}@{}".format(
m[1].market_start_time, m[1].event.venue))
event_mapping[m[0]] = e, m[1].market_start_time, m[
1].event.country_code, mb_market
bf_selection_name_mapping.update(
{r.selection_id: r.runner_name
for r in m[1]['runners']})
for p in tqdm(prices):
# find turf bookmaker odds
turf_odds_df = pd.DataFrame()
race_status = {}
for m_id, m in markets[event_mapping[p.market_id][0]]:
if m_id != p.market_id:
continue
market_start_time = event_mapping[p.market_id][1]
venue = m.event.venue.lower()
config = get_config()
turf_countries = config.get('Scraping', 'turf_countries').split()
if m.event.country_code not in turf_countries:
continue
# TODO, check and process turf odds
try:
turf_odds_df, race_status = get_race_odds(
venue, market_start_time, m.runners)
except:
pass
mb_market = event_mapping[p.market_id][3]
for r in p.runners:
back_prices, back_sizes, lay_prices, lay_sizes = [-1, -1, -1], [
-1, -1, -1
], [-1, -1, -1], [-1, -1, -1]
try:
vwap = get_VWAP(r.ex.traded_volume)
except ZeroDivisionError:
vwap = 0.
for i in range(3):
try:
back_prices[i] = r.ex.available_to_back[i].price
except IndexError:
back_prices[i] = None
try:
lay_prices[i] = r.ex.available_to_lay[i].price
except IndexError:
lay_prices[i] = None
try:
back_sizes[i] = r.ex.available_to_back[i].size
except IndexError:
back_sizes[i] = None
try:
lay_sizes[i] = r.ex.available_to_lay[i].size
except IndexError:
lay_sizes[i] = None
try:
sp_dict = {
'near_price':
r.sp.near_price,
'far_price':
r.sp.far_price,
'actual_SP':
r.sp.actual_SP,
'back_SP_amounts':
[(a.price, a.size) for a in r.sp.back_stake_taken],
'lay_SP_amounts':
[(a.price, a.size) for a in r.sp.back_stake_taken]
}
except:
sp_dict = {}
seconds_until_start = (event_mapping[p.market_id][1] -
datetime.datetime.now()).total_seconds()
event_id = event_mapping[p.market_id][0]
d = {
'marketid': p.market_id,
'selection_id': r.selection_id,
'LTP': r.last_price_traded,
'back_sizes0': back_sizes[0],
'back_prices0': back_prices[0],
'back_sizes1': back_sizes[1],
'back_prices1': back_prices[1],
'back_sizes2': back_sizes[2],
'back_prices2': back_prices[2],
'lay_sizes0': lay_sizes[0],
'lay_prices0': lay_prices[0],
'lay_sizes1': lay_sizes[1],
'lay_prices1': lay_prices[1],
'lay_sizes2': lay_sizes[2],
'lay_prices2': lay_prices[2],
'timestamp': datetime.datetime.now(),
'eventid': event_id,
'marketstarttime': event_mapping[p.market_id][1],
'countrycode': event_mapping[p.market_id][2],
'seconds_until_start': seconds_until_start,
'total_matched': r.total_matched,
'VWAP': vwap,
}
d.update(sp_dict)
if not turf_odds_df.empty:
# Add the turf odds
try:
bookie_prices = turf_odds_df.loc[
r.selection_id].values.astype(float)
bookie_prices[bookie_prices ==
0] = np.nan # replace 0 with nan
d['bookies'] = pd.DataFrame({'name': turf_odds_df.loc[r.selection_id].index,
'price': bookie_prices}). \
to_dict(orient='records)')
d['mean_bookie_price'] = np.nanmean(bookie_prices)
d['median_bookie_price'] = np.nanmedian(bookie_prices)
d['min_bookie_price'] = np.nanmin(bookie_prices)
d['max_bookie_price'] = np.nanmax(bookie_prices)
except KeyError:
log.info("No turf odds for selection: {}".format(
r.selection_id))
if mb_market:
d['mb_market_id'] = mb_market['id']
runner_name = bf_selection_name_mapping[r.selection_id]
mb_selection_id = None
for mr in mb_market['runners']:
if mr['name'].lstrip('1234567890- ') == runner_name:
mb_selection_id = mr['id']
d['mb_selection_id'] = mb_selection_id
d['mb_volume'] = mr['volume']
back_index, lay_index = 0, 0
for mp in mr['prices']:
if mp['side'] == 'back':
d['mb_back_prices{}'.format(
back_index)] = mp['decimal-odds']
d['mb_back_sizes{}'.format(
back_index)] = mp['available-amount']
back_index += 1
else:
d['mb_lay_prices{}'.format(
back_index)] = mp['decimal-odds']
d['mb_lay_sizes{}'.format(
back_index)] = mp['available-amount']
lay_index += 1
if not mb_selection_id:
log.warning(
"Didn't find runner information for bf selection id {}"
.format(r.selection_id))
d.update(race_status)
log.debug(d)
m = MongoManager()
success = m.insert_document(collection_name, d)
log.debug(success)
def __init__(self, use_remote=True, use_archive=False):
# to create a local copy, run mongod locally with shell and copy over the remote with this command in mongodb
# db.copyDatabase("horse_racing","horse_racing","dickreuter.com","rj","rj","SCRAM-SHA-1" )
# to run it automatically as service in windows, do this with admin rights: mongod --install
config = get_config()
prod = config.get("Servers", "prod")
remote_server = config.get("MongoDB", "remote_server")
# use localhost when program runs on prod:
localhost = config.get("MongoDB", "localhost")
# database for price operations
price_db = config.get("MongoDB", "price_db")
price_login = config.get("MongoDB", "price_login")
price_pass = config.get("MongoDB", "price_pass")
price_login_write = config.get("MongoDB", "price_login_write")
price_pass_write = config.get("MongoDB", "price_pass_write")
# database for pnl operations
pnl_db = config.get("MongoDB", "pnl_db")
pnl_login = config.get("MongoDB", "pnl_login")
pnl_pass = config.get("MongoDB", "pnl_pass")
pnl_login_write = config.get("MongoDB", "pnl_login_write")
pnl_pass_write = config.get("MongoDB", "pnl_pass_write")
# database for archive
archive_db = config.get("MongoDB", "archive_db")
archive_login_write = config.get("MongoDB", "archive_login_write")
archive_pass_write = config.get("MongoDB", "archive_pass_write")
if use_archive: # use archive for everything
# database for price operations
price_db = archive_db
price_login = archive_login_write
price_pass = archive_pass_write
price_login_write = archive_login_write
price_pass_write = archive_pass_write
# database for pnl operations
pnl_db = archive_db
pnl_login = archive_login_write
pnl_pass = archive_pass_write
pnl_login_write = archive_login_write
pnl_pass_write = archive_pass_write
if use_remote:
if hostname == prod:
self.mongoclient_price = MongoClient(
'mongodb://{}:{}@{}/{}'.format(price_login_write,
price_pass_write, localhost,
price_db))
self.mongoclient_pnl = MongoClient(
'mongodb://{}:{}@{}/{}'.format(pnl_login_write,
pnl_pass_write, localhost,
pnl_db))
self.mongoclient_archive = MongoClient(
'mongodb://{}:{}@{}/{}'.format(archive_login_write,
archive_pass_write,
localhost, archive_db))
else:
self.mongoclient_price = MongoClient(
'mongodb://{}:{}@{}/{}'.format(price_login, price_pass,
remote_server, price_db))
self.mongoclient_pnl = MongoClient(
'mongodb://{}:{}@{}/{}'.format(pnl_login, pnl_pass,
remote_server, pnl_db))
self.mongoclient_archive = MongoClient(
'mongodb://{}:{}@{}/{}'.format(archive_login_write,
archive_pass_write,
remote_server, archive_db))
else:
self.mongoclient_price = MongoClient(
'mongodb://localhost/horse_racing')
self.mongoclient_pnl = MongoClient(
'mongodb://localhost/horse_racing')
self.mongoclient_archive = MongoClient(
'mongodb://localhost/archive')
self.mongodb_price = self.mongoclient_price[price_db]
self.mongodb_pnl = self.mongoclient_pnl[pnl_db]
self.mongodb_archive = self.mongoclient_archive[archive_db]