def report_outcome_for_football(self, response):
for key, orders in self.orders_by_provider.iteritems():
for bet_info in orders:
sport, (_, event_id), market, params, bm = parse_sticker(
bet_info.sticker)
gsm_id = key[0]
try:
details = response[gsm_id]['details']
except KeyError:
logging.debug("Manual #Order %s" % bet_info)
continue
last_score_ft = (details['team1FTG'], details['team2FTG'])
last_score_ht = (details['team1HTG'], details['team2HTG'])
if market in (Markets.FULL_TIME_ASIAN_HANDICAP_GOALS,
Markets.FULL_TIME_OVER_UNDER_GOALS,
Markets.FULL_TIME_CORRECT_SCORE,
Markets.FULL_TIME_1X2):
n_win, n_bet = determine_football_outcome(
market, params, last_score_ft)
elif market in (Markets.HALF_TIME_ASIAN_HANDICAP_GOALS,
Markets.HALF_TIME_OVER_UNDER_GOALS,
Markets.HALF_TIME_CORRECT_SCORE,
Markets.HALF_TIME_1X2):
n_win, n_bet = determine_football_outcome(
market, params, last_score_ft, last_score_ht)
else:
logging.error("Cannot determine market pnl #Order %s\n" %
bet_info)
continue
report_bet_outcome_pnl(n_win, n_bet, bet_info)
def football_pnl_grid(orders, normalize_by_stake=False, max_evts=7, default_if_unknown=False, reshape=False):
'''
:param orders: can be for (e.g.) goal or corner markets
:param max_evts: the maximum number of events (e.g. goals or corners) or a team handled by the pnl grid
'''
pnl_per_score = []
aggregate_bet_notional = 1
if normalize_by_stake:
aggregate_bet_notional = sum(order_size(b, default_if_unknown) for b in orders)
for team1_score, team2_score in product(range(max_evts + 1), range(max_evts + 1)):
pnl_for_score = 0
for order in orders:
_, market_scope, market, params, _ = parse_sticker(order.sticker)
n_win, n_bet = determine_football_outcome(market, params, (team1_score, team2_score))
outcome, pnl = determine_order_outcome_pnl(order, n_win, n_bet, default_if_unknown=default_if_unknown)
if normalize_by_stake:
pnl /= float(aggregate_bet_notional)
pnl_for_score += pnl
pnl_per_score.append(pnl_for_score)
if reshape:
return np.array(pnl_per_score).reshape(max_evts + 1, max_evts + 1)
return np.array(pnl_per_score)
def report_outcome_for_tennis(self, response):
for key, orders in self.orders_by_provider.iteritems():
for bet_info in orders:
sport, (_, event_id), market, params, bm = parse_sticker(
bet_info.sticker)
enp_id = key[0]
try:
details = response[enp_id]['details']
except KeyError:
logging.debug("Manual #Order %s" % bet_info)
continue
player_a_result = details.get('playerAResult', -1)
if market == TennisMarkets.MATCH_ODDS:
n_win, n_bet = determine_tennis_outcome(
market, params, player_a_result)
report_bet_outcome_pnl(n_win, n_bet, bet_info)
elif market in [
TennisMarkets.SET_BETTING,
TennisMarkets.TOTAL_GAMES_OVER_UNDER,
TennisMarkets.HANDICAP_GAMES
]:
scores = response.get(enp_id, {}).get('details', {})
n_win, n_bet = determine_tennis_outcome_alternative_markets(
market, params, scores)
report_bet_outcome_pnl(n_win, n_bet, bet_info)
else:
logging.error("Cannot determine market pnl #Order %s\n" %
bet_info)
continue
def market_data_stickers_figures_for_trade_id(self, trade_id, stickers_for_trade_id):
# get all historical odds for the stickers
# sticker is
best_odds_figs = []
for sticker in stickers_for_trade_id:
sport, market_scope, market, params, _ = parse_sticker(sticker)
stream = OddsStream(sport, market_scope[1], market, *params,
bookmaker=Bookmakers.BETFAIR)
odds = get_historical_odds([stream])[stream]
# Calculate best back and lay for a unique sticker
back_dt = []
lay_dt = []
back_odds = []
lay_odds = []
for oddsTick in odds:
if len(oddsTick.back) > 0:
back_odds.append(oddsTick.back[0].o)
back_dt.append(oddsTick.timestamp)
if len(oddsTick.lay) > 0:
lay_odds.append(oddsTick.lay[0].o)
lay_dt.append(oddsTick.timestamp)
# best odds fig
best_odds_fig = self.construct_best_odds_figure(back_dt, back_odds, lay_dt, lay_odds, trade_id, sticker)
best_odds_figs.append(best_odds_fig)
return best_odds_figs
def run(self):
for trading_user_id in TRADING_USERS.values():
for strategy_name in self.strategy_names:
orders = get_settled_orders(trading_user_id, self.start_dt,
self.end_dt, strategy_name)
for order in orders:
try:
bet_info = json_to_bet_info(order)
except KeyError:
logging.error("Missing Sticker #Order %s\n" % order)
continue
sport, (scope,
event_id), market, params, bm = parse_sticker(
bet_info.sticker)
exec_details = bet_info.execution_details
provider = exec_details['provider']
if provider in self.providers:
bookmaker = exec_details['bookmaker']
sticker = bet_info.sticker
if len(sticker) == 0:
logging.error("Missing Sticker #Order %s\n" %
order)
continue
if scope == MarketScopes.EVENT:
# just event scopes for now
current_orders = self.orders_by_provider[event_id,
provider,
bookmaker]
current_orders.append(bet_info)
if len(self.orders_by_provider
) >= _FLUSH_NUM_ORDERS:
# Calculate outcomes for all event_ids
event_ids = self._get_event_ids()
response = data_api.get_event_outcome(
sport, event_ids)
self.report_outcome_for_events(response)
self.report_missing_fx_rate()
# flush dictionary
self.orders_by_provider.clear()
self.flushed = True
if self.orders_by_provider:
# Calculate outcomes for all event_ids
event_ids = self._get_event_ids()
response = data_api.get_event_outcome(
extract_sport(bet_info.sticker), event_ids)
self.report_outcome_for_events(response)
self.report_missing_fx_rate()
elif not self.flushed:
logging.info("No orders found")
def tennis_pnl_grid_match_winner(orders):
"""
Returns a 1d array containing the pnl in case player A or player B wins the match. Only works with MATCH ODDS
market
"""
pnl_per_result = np.zeros(2)
for idx, player_a_res in enumerate(['winner', 'loser']):
for order in orders:
_, _, _, params, _ = parse_sticker(order.sticker)
nwin, n_bet = determine_tennis_outcome(
TennisMarkets.MATCH_ODDS, params, player_a_res)
outcome, pnl = determine_order_outcome_pnl(
order, nwin, n_bet, default_if_unknown=False)
pnl_per_result[idx] += pnl
return pnl_per_result
def get_per_event_vol(self, stats, instructions, orders):
per_event_vol = {}
for order in orders:
pnl = StrategyRunStatsHelper.get_order_matched_vol(order)
if pnl == 0.:
continue
sport, (scope,
event_id), mkt, params, _ = parse_sticker(order['sticker'])
if event_id not in per_event_vol:
per_event_vol[event_id] = 0.
per_event_vol[event_id] += pnl
for key in per_event_vol.keys():
per_event_vol[key] = round(per_event_vol[key], 2)
return per_event_vol
def f_instruction_to_row(sport, instr, extra_info=None):
'''
:param extra_info: (dict) extra information to take from raw instructions,
e.g. {'timestamp': 'details.signals.0.timestamp'}
'''
_, (_, event_id), mkt_id, params, _ = parse_sticker(instr['sticker'])
sel_id = params[0]
hc = np.nan if len(params) == 1 else params[1]
instr_dict = {
'fixture_id': int(event_id[3:]),
'average_price_matched': instr['average_price_matched'],
'capital_received': instr['details'].get('capital_received', -1),
'sticker': instr['sticker'],
'market': MARKET_ABBR[sport][mkt_id],
'selection': Selections.to_str(sel_id),
'hc': hc,
'is_back': (instr['bet_side'] == 'back'),
'instruction_id': instr['id'],
'selection_id': sel_id,
'market_id': mkt_id,
'handicap': hc,
'placed_time': parse_date_if_necessary(instr['placed_time'],
to_utc=True),
'size_wanted': instr['size'],
'size_matched': instr['size_matched'],
'strategy': instr['strategy'],
'strategy_descr': instr['strategy_descr'],
'trade_id': instr['trade_id'],
}
if extra_info is not None:
# get specified additional information from instruction
info_dict = {} # initialise
for label, i in extra_info.iteritems():
i_keys = i.split('.')
info = instr.copy() # initialise
for i_key in i_keys:
try:
i_key = int(i_key) # convert to int if possible
except:
pass
info = info[i_key] # update
info_dict[label] = info
instr_dict.update(info_dict)
return instr_dict
def _add_sticker_info(self, sticker, order, info):
sport, (market_scope,
gsm_id), market_id, params, bm = parse_sticker(sticker)
if sport != self._sport:
raise Exception('Received a sticker which is not from the correct'
' sport {} ! {}'.format(self._sport, order))
sticker_info = {
'sticker': sticker,
'event_id': gsm_id,
'fixture_id': int(gsm_id[3:]),
'handicap': None if len(params) == 1 else params[1],
'selection_id': params[0],
'selection': SELECTION_ABBR[self._sport][params[0]],
'market': MARKET_ABBR[self._sport][market_id],
'market_id': market_id
}
info.update(sticker_info)
def report_outcome_for_basketball(self):
for key, orders in self.orders_by_provider.iteritems():
for bet_info in orders:
sport, (_, event_id), market, params, bm = parse_sticker(
bet_info.sticker)
enp_id = key[0]
if market in (BasketballMarkets.FULL_TIME_POINT_SPREAD,
BasketballMarkets.FULL_TIME_MONEYLINE,
BasketballMarkets.FULL_TIME_TOTAL_POINTS):
n_win, n_bet = determine_basketball_outcome_from_api(
market, params, enp_id)
report_bet_outcome_pnl(n_win, n_bet, bet_info)
else:
logging.error("Cannot determine market pnl #Order %s\n" %
bet_info)
continue
def convert_bet_states_to_array(order_states):
"""
Convert bet_states DataFrame to array of bets containing only last known bet state
:param order_states: DataFrame of bet states
:return: array of bet objects
"""
orders = dict()
for timestamp, order in order_states.iteritems():
_, market_scope, market, params, _ = parse_sticker(order.sticker)
orders[order.id] = {
'event': market_scope[1],
'pnl': order.pnl,
'market': market,
'commission': order.commission,
'stake': order.size,
'is_back': order.is_back,
'odds': order.price if order.matched_odds == 0 else order.matched_odds,
'date': timestamp,
}
return orders.values()
def get_order_bm(order):
sport, (scope,
event_id), mkt, params, bm = parse_sticker(order['sticker'])
if bm is not None:
return BOOKMAKER_ABBR[bm]
return order['execution_details']['bookmaker']
def _update_max_loss(self, updated_instructions, _updated_orders):
closed_trades = set()
for instruction in updated_instructions:
sport, (_, event_id), market, params, bm = parse_sticker(
instruction.sticker)
trade_id = instruction.trade_id
if sport == Sports.FOOTBALL:
# if not already present, initialise pnl grids
# (accounting for the possibility of a corner market, which affects the size of the grids)
# note: this effectively replaces the default grid of the defaultdict
max_evts = _get_football_max_evts(instruction)
if event_id not in self._current_pnl_grid_by_trade_by_event:
self._current_pnl_grid_by_trade_by_event[event_id] = {}
if trade_id not in self._current_pnl_grid_by_trade_by_event[
event_id]:
self._current_pnl_grid_by_trade_by_event[event_id][
trade_id] = empty_pnl_grid(grid_size=max_evts + 1)
if event_id not in self._active_pnl_grid_by_trade_by_event:
self._active_pnl_grid_by_trade_by_event[event_id] = {}
if trade_id not in self._active_pnl_grid_by_trade_by_event[
event_id]:
self._active_pnl_grid_by_trade_by_event[event_id][
trade_id] = empty_pnl_grid(grid_size=max_evts + 1)
else:
err_msg = ('WARN: risk not implemented for sport {}'.format(
instruction.sticker))
warnings.warn(err_msg)
if instruction in self._instructions_by_trade_by_event[event_id][
trade_id]:
self._current_pnl_grid_by_trade_by_event[event_id][
trade_id] -= self._current_pnl_grid_by_instruction_id[
instruction.id]
self._active_pnl_grid_by_trade_by_event[event_id][
trade_id] -= self._active_pnl_grid_by_instruction_id[
instruction.id]
self._instructions_by_trade_by_event[event_id][trade_id].add(
instruction)
# create an instruction for the total matchable amount
instruction_copy = deepcopy(instruction)
if instruction.status in (InstructionStatus.IDLE,
InstructionStatus.PROCESSING,
InstructionStatus.WATCHING):
instruction_copy.matched_amount = instruction.size
else:
instruction_copy.matched_amount = instruction.matched_amount
sport = extract_sport(instruction.sticker)
if sport == Sports.FOOTBALL:
max_evts = _get_football_max_evts(instruction)
self._current_pnl_grid_by_instruction_id[
instruction.id] = football_pnl_grid([instruction],
max_evts=max_evts,
reshape=True)
self._active_pnl_grid_by_instruction_id[
instruction.id] = football_pnl_grid([instruction_copy],
max_evts=max_evts,
reshape=True)
else:
# TODO more sports
self._current_pnl_grid_by_instruction_id[
instruction.id] = empty_pnl_grid()
self._active_pnl_grid_by_instruction_id[
instruction.id] = empty_pnl_grid()
self._current_pnl_grid_by_trade_by_event[event_id][
trade_id] += self._current_pnl_grid_by_instruction_id[
instruction.id]
self._active_pnl_grid_by_trade_by_event[event_id][
trade_id] += self._active_pnl_grid_by_instruction_id[
instruction.id]
# check if a trade has opened, the open dt will be that provided by the strategy,
# this is for RINA style calculations - this is now the only way
# to calculate time in market, there is no fancy attempts to automatically calculate it
if trade_id not in self._trade_open_dt and 'trade_open_dt' in instruction.details:
trade_open_dt = instruction.details['trade_open_dt']
self._trade_open_dt[trade_id] = trade_open_dt
# check if a trade has been labelled closed by the strategy
if trade_id in self._trade_open_dt and trade_id not in self._trade_close_dt and 'trade_close_dt' in instruction.details:
trade_closed_dt = instruction.details['trade_close_dt']
self._trade_close_dt[trade_id] = trade_closed_dt
closed_trades.add(trade_id)
# TODO consider reducing max loss as orders settle?
max_loss = 0
# calculate the max loss from the active pnl grids, ie the total matchable amounts
for event_id, pnl_grid_by_trade in self._active_pnl_grid_by_trade_by_event.items(
):
max_loss_for_event = 0
for trade_id, pnl_grid_for_trade in pnl_grid_by_trade.items():
min_pnl_for_trade = np.min(pnl_grid_for_trade)
max_loss_for_trade = min(0, min_pnl_for_trade)
self._max_loss_by_trade_by_event[event_id][
trade_id] = max_loss_for_trade
max_loss_for_event += max_loss_for_trade
max_loss += max_loss_for_event
self._max_loss = max_loss
for closed_trade_id in closed_trades:
if closed_trade_id not in self._time_in_market_by_trade:
time_in_market_sec = (
self._trade_close_dt[closed_trade_id] -
self._trade_open_dt[closed_trade_id]).total_seconds()
time_in_market = min(1., time_in_market_sec / (111. * 60.))
self._time_in_market_by_trade[closed_trade_id] = time_in_market
new_mean_time_in_market = get_new_mean(
self._mean_time_in_market, time_in_market,
len(self._time_in_market_by_trade))
self._mean_time_in_market = new_mean_time_in_market
def _get_football_max_evts(instruction):
_, _, market, _, _ = parse_sticker(instruction.sticker)
return (25 if Markets.is_corner_mkt(market) else
_get_default_args(football_pnl_grid)['max_evts'])
def execution_data_tennis_sip(trading_user_id,
start_dt,
end_dt,
strategy_id,
strategy_run_id=None):
cache = HistoricalOddsCache()
mysql_enet = MySQLClient.init_from_config(auto_connect=True)
orders = get_settled_orders(trading_user_id, start_dt, end_dt, strategy_id,
strategy_run_id)
instruction_ids = {
order['instruction_id']
for order in orders if 'instruction_id' in order
}
instructions = get_instructions_by_id(trading_user_id,
list(instruction_ids))
stickers = {
order['sticker']
for order in orders if len(order['sticker']) > 0
}
enp_ids = set()
for sticker in stickers:
sport, (_, event_id), market, params, _ = parse_sticker(sticker)
enp_ids.add(event_id[3:])
if len(enp_ids) == 0:
return []
results_query = _SQL_QUERY % ', '.join(enp_ids)
results_tz = pytz.timezone('Europe/London')
try:
event_action_data = mysql_enet.select(results_query, as_list=True)
except:
print results_query
raise
set_end_times = get_set_end_times(event_action_data)
rows = []
for order in orders:
sticker = order['sticker']
rate = order['exchange_rate']
if len(sticker) == 0:
continue
sport, (_, event_id), market, params, _ = parse_sticker(sticker)
start_dt_end_set1 = results_tz.localize(
set_end_times[event_id]['set1'])
start_dt_end_set2 = results_tz.localize(
set_end_times[event_id]['set2'])
placed_dt = ciso8601.parse_datetime(order['placed_time'])
if start_dt_end_set1 - timedelta(
minutes=5) < placed_dt < start_dt_end_set2:
bf_vwap = exchange_vwap(sticker,
Bookmakers.BETFAIR,
start_dt_end_set1,
start_dt_end_set2,
odds_cache=cache,
return_nan=True)
suffix = 'set2'
else:
if 'set3' not in set_end_times[event_id]:
bf_vwap = float('nan')
suffix = 'unknown'
else:
start_dt_end_set3 = results_tz.localize(
set_end_times[event_id]['set3'])
if start_dt_end_set2 - timedelta(
minutes=5) < placed_dt < start_dt_end_set3:
bf_vwap = exchange_vwap(sticker,
Bookmakers.BETFAIR,
start_dt_end_set2,
start_dt_end_set3,
odds_cache=cache,
return_nan=True)
suffix = 'set3'
else:
bf_vwap = float('nan')
suffix = 'unknown'
try:
rows.append({
'order_id':
order['id'],
'order_source':
order['source'],
'instruction_id':
order.get('instruction_id', ''),
'strategy_descr':
order['strategy_descr'],
'sticker':
sticker,
'order_size':
order['size'] / rate,
'source':
order['source'],
'matched_size':
order['size_matched'] / rate,
'pnl':
order['outcome']['net'] / rate,
'limit_price':
order['price'],
'average_price':
order['average_price_matched'],
'side':
order['bet_side'],
'bf_vwap_%s' % suffix:
bf_vwap,
'bookmaker':
order['execution_details'].get('bookmaker', ''),
'benchmark_price':
bf_vwap,
})
except:
print order
raise
return rows
instructions = data_api.get_closed_instructions(trading_user_id,
strategy_id)
filtered_instructions = []
for instr_json in instructions:
if start_dt <= parser.parse(instr_json['placed_time']) <= end_dt:
filtered_instructions.append(instr_json)
for instr_json in filtered_instructions:
sticker = instr_json['sticker']
if instr_json['size_matched'] < instr_json['size']:
remaining = instr_json['size'] - instr_json['size_matched']
sport, market_scope, market, params, _ = parse_sticker(sticker)
outcome, n_bet = determine_tennis_outcome_from_api(
market, params, market_scope[1])
placed_dt = parser.parse(instr_json['placed_time'])
vwap = exchange_vwap(sticker,
Bookmakers.BETFAIR,
placed_dt,
placed_dt + timedelta(seconds=50),
odds_cache=cache,
return_nan=True)
instr = json_to_instruction(instr_json)
instr.size = remaining
instr.matched_amount = 0
