def prep_df(ohlcvs: pd.DataFrame, settings: dict) -> pd.DataFrame: ema_spans = settings['ema_spans_minutes'] entry_spread = settings['entry_spread'] precision = calc_price_precision(ohlcvs.close) emas = pd.concat([ pd.Series(ohlcvs.close.ewm(span=span, adjust=False).mean(), name=str(span)) for span in ema_spans ], axis=1) min_ema = emas.min(axis=1) max_ema = emas.max(axis=1) entry_bid = round_dn(min_ema * (1 - entry_spread), precision) entry_ask = round_up(max_ema * (1 + entry_spread), precision) exit_bid = round_dn(min_ema, precision) exit_ask = round_up(max_ema, precision) avg = ohlcvs[['open', 'close']].mean(axis=1) df = pd.DataFrame( { 'entry_bid': entry_bid, 'entry_ask': entry_ask, 'exit_bid': exit_bid, 'exit_ask': exit_ask, 'avg': avg, 'high': ohlcvs.high, 'low': ohlcvs.low }, index=ohlcvs.index) return df[(df.low < df.exit_bid) | (df.high > df.exit_ask)]
def create_df(symbols: [str], n_days: int, settings: dict, no_download: bool = False): dfs = [] for s in symbols: coin, quot = s.split('/') print('preparing', s) rt = trade_data.fetch_raw_trades(s, n_days=n_days, no_download=no_download) print('precision') precision = find_precision(rt.price) print('emas') start_ts = time() rt_emas = calc_ema_from_raw_trades(rt, settings['ema_spans_minutes']) print('elapsed seconds calc emas', round(time() - start_ts, 2)) ema_min = rt_emas.min(axis=1) ema_max = rt_emas.max(axis=1) long_entry = round_dn( ema_min * (1 - settings['coins'][coin]['entry_spread']), precision) shrt_entry = round_up( ema_max * (1 + settings['coins'][coin]['entry_spread']), precision) long_exit = round_up(ema_max, precision) shrt_exit = round_dn(ema_min, precision) rt.loc[:, 'entry_price'] = long_entry.where(rt.is_buyer_maker, shrt_entry) rt.loc[:, 'exit_price'] = long_exit.where(~rt.is_buyer_maker, shrt_exit) rt.loc[:, 'entry'] = ((rt.is_buyer_maker & (rt.price < long_entry)) | (~rt.is_buyer_maker & (rt.price > shrt_entry))) rt = rt[((rt.is_buyer_maker & (rt.price < shrt_exit)) | (~rt.is_buyer_maker & (rt.price > long_exit)))] rt.loc[:, 'symbol'] = np.repeat(s, len(rt)) dfs.append(rt.set_index('timestamp')) return pd.concat(dfs, axis=0).sort_index()
def backtest(df: pd.DataFrame, settings: dict, price_precisions: dict = {}): start_quot = 1.0 ppctminus = 1 - settings['profit_pct'] ppctplus = 1 + settings['profit_pct'] symbols = [c.replace('_low', '') for c in df.columns if 'low' in c] if not price_precisions: price_precisions = {s: 8 for s in symbols} lows = {s: f'{s}_low' for s in symbols} highs = {s: f'{s}_high' for s in symbols} means = {s: f'{s}_mean' for s in symbols} min_emas = {s: f'{s}_mean_min_ema' for s in symbols} max_emas = {s: f'{s}_mean_max_ema' for s in symbols} min_delay_millis = settings['min_seconds_between_same_side_entries'] * 1000 rolling_millis = settings['max_memory_span_days'] * 24 * 60 * 60 * 1000 s2c = {s: s.split('_')[0] for s in symbols} quot = symbols[0].split('_')[1] balance = {s2c[s]: 0.0 for s in s2c} balance[quot] = 1.0 acc_equity_quot = 1.0 acc_debt_quot = 0.0 long_entries = {s: [] for s in symbols} shrt_entries = {s: [] for s in symbols} long_exits = {s: [] for s in symbols} shrt_exits = {s: [] for s in symbols} long_exit_price_list = {s: [] for s in symbols} shrt_exit_price_list = {s: [] for s in symbols} past_rolling_long_entries = {s: [] for s in symbols} past_rolling_shrt_entries = {s: [] for s in symbols} entry_bid = {s: round(df.iloc[0][means[s]], 8) for s in symbols} entry_ask = {s: round(df.iloc[0][means[s]], 8) for s in symbols} exit_bid = {s: entry_bid[s] for s in symbols} exit_ask = {s: entry_ask[s] for s in symbols} long_cost = {s: 0.0 for s in symbols} long_amount = {s: 0.0 for s in symbols} shrt_cost = {s: 0.0 for s in symbols} shrt_amount = {s: 0.0 for s in symbols} fee = 1 - 0.000675 # vip 1 margin_level = 3 - 1 balance_list = [] do_shrt = {s for s in symbols if s2c[s] in settings['coins_shrt']} do_long = {s for s in symbols if s2c[s] in settings['coins_long']} exponent = settings['entry_vol_modifier_exponent'] start_ts, end_ts = df.index[0], df.index[-1] ts_range = end_ts - start_ts for row in df.itertuples(): cost = acc_equity_quot * settings['account_equity_pct_per_trade'] min_exit_cost = cost * settings['min_big_trade_cost_multiplier'] credit_avbl_quot = max(0.0, acc_equity_quot * margin_level - acc_debt_quot) age_limit = row.Index - rolling_millis for s in symbols: # rolling longs long_i = get_cutoff_index(past_rolling_long_entries[s], age_limit) if long_i > 0: slc = past_rolling_long_entries[s][:long_i] past_rolling_long_entries[s] = past_rolling_long_entries[s][long_i:] long_amount[s] -= sum([e['amount'] for e in slc]) long_cost[s] -= sum([e['amount'] * e['price'] for e in slc]) if long_cost[s] <= 0.0 or long_amount[s] <= 0.0: long_cost[s] = 0.0 long_amount[s] = 0.0 past_rolling_long_entries[s] = [] exit_ask[s] = getattr(row, means[s]) else: exit_ask[s] = (long_cost[s] / long_amount[s]) * ppctplus # rolling shrts shrt_i = get_cutoff_index(past_rolling_shrt_entries[s], age_limit) if shrt_i > 0: slc = past_rolling_shrt_entries[s][:shrt_i] past_rolling_shrt_entries[s] = past_rolling_shrt_entries[s][shrt_i:] shrt_cost[s] -= sum([e['amount'] * e['price'] for e in slc]) shrt_amount[s] -= sum([e['amount'] for e in slc]) if shrt_cost[s] <= 0.0 or shrt_amount[s] <= 0.0: shrt_cost[s] = 0.0 shrt_amount[s] = 0.0 past_rolling_shrt_entries[s] = [] exit_bid[s] = getattr(row, means[s]) else: exit_bid[s] = (shrt_cost[s] / shrt_amount[s]) * ppctminus if s in do_long and getattr(row, lows[s]) < entry_bid[s] and \ (not long_entries[s] or (row.Index - long_entries[s][-1]['timestamp'] >= min_delay_millis)): # long buy long_modifier = max( 1.0, min(settings['min_big_trade_cost_multiplier'] - 1, (exit_ask[s] / getattr(row, means[s]))**exponent)) buy_cost = cost * long_modifier if balance[quot] >= buy_cost: # long buy normal buy_amount = (buy_cost / entry_bid[s]) balance[quot] -= buy_cost balance[s2c[s]] += buy_amount * fee long_entries[s].append({'price': entry_bid[s], 'amount': buy_amount, 'timestamp': row.Index}) past_rolling_long_entries[s].append(long_entries[s][-1]) long_amount[s] += buy_amount long_cost[s] += buy_cost exit_ask[s] = (long_cost[s] / long_amount[s]) * ppctplus elif credit_avbl_quot > 0.0: # long buy with credit quot_avbl = max(0.0, balance[quot]) to_borrow = min(credit_avbl_quot, buy_cost - quot_avbl) credit_avbl_quot -= to_borrow partial_buy_cost = quot_avbl + to_borrow buy_amount = (partial_buy_cost / entry_bid[s]) balance[quot] -= partial_buy_cost balance[s2c[s]] += buy_amount * fee long_entries[s].append({'price': entry_bid[s], 'amount': buy_amount, 'timestamp': row.Index}) past_rolling_long_entries[s].append(long_entries[s][-1]) long_amount[s] += buy_amount long_cost[s] += partial_buy_cost exit_ask[s] = (long_cost[s] / long_amount[s]) * ppctplus if s in do_shrt and getattr(row, highs[s]) > entry_ask[s] and \ (not shrt_entries[s] or (row.Index - shrt_entries[s][-1]['timestamp'] >= min_delay_millis)): # shrt sel shrt_modifier = max( 1.0, min(settings['min_big_trade_cost_multiplier'] - 1, (getattr(row, means[s]) / exit_bid[s])**exponent)) sel_cost = cost * shrt_modifier sel_amount = sel_cost / entry_ask[s] if balance[s2c[s]] >= sel_amount: # shrt sel normal balance[s2c[s]] -= sel_amount balance[quot] += sel_cost * fee shrt_entries[s].append({'price': entry_ask[s], 'amount': sel_amount, 'timestamp': row.Index}) past_rolling_shrt_entries[s].append(shrt_entries[s][-1]) shrt_amount[s] += sel_amount shrt_cost[s] += sel_cost exit_bid[s] = (shrt_cost[s] / shrt_amount[s]) * ppctminus elif credit_avbl_quot > 0.0: # shrt sel with credit coin_avbl = max(0.0, balance[s2c[s]]) to_borrow = min(credit_avbl_quot / entry_ask[s], sel_amount - coin_avbl) credit_avbl_quot -= (to_borrow * entry_ask[s]) partial_sel_amount = coin_avbl + to_borrow balance[s2c[s]] -= partial_sel_amount partial_sel_cost = partial_sel_amount * entry_ask[s] balance[quot] += partial_sel_cost * fee shrt_entries[s].append({'price': entry_ask[s], 'amount': partial_sel_amount, 'timestamp': row.Index}) past_rolling_shrt_entries[s].append(shrt_entries[s][-1]) shrt_amount[s] += partial_sel_amount shrt_cost[s] += partial_sel_cost exit_bid[s] = (shrt_cost[s] / shrt_amount[s]) * ppctminus exit_ask[s] = round_up(exit_ask[s], price_precisions[s]) exit_bid[s] = round_dn(exit_bid[s], price_precisions[s]) if long_cost[s] > min_exit_cost: # long sel long_exit_price_list[s].append({'price': exit_ask[s], 'timestamp': row.Index}) if getattr(row, highs[s]) > exit_ask[s]: if balance[s2c[s]] >= long_amount[s]: # long sel normal long_sel_amount = max(balance[s2c[s]], long_amount[s]) long_exits[s].append({'price': exit_ask[s], 'amount': long_sel_amount, 'timestamp': row.Index}) quot_acquired = long_sel_amount * exit_ask[s] balance[s2c[s]] -= long_sel_amount balance[quot] += quot_acquired * fee long_amount[s] = 0.0 long_cost[s] = 0.0 else: # partial long sel coin_avbl = max(0.0, balance[s2c[s]]) to_borrow = min(credit_avbl_quot / exit_ask[s], long_amount[s] - coin_avbl) partial_sel_amount = coin_avbl + to_borrow if partial_sel_amount > 0.0: credit_avbl_quot -= (to_borrow * exit_ask[s]) balance[s2c[s]] -= partial_sel_amount partial_sel_cost = partial_sel_amount * exit_ask[s] balance[quot] += partial_sel_cost * fee long_exits[s].append({'price': exit_ask[s], 'amount': partial_sel_amount, 'timestamp': row.Index}) long_amount[s] -= partial_sel_amount long_cost[s] -= partial_sel_cost if long_amount[s] <= 0.0 or long_cost[s] <= 0.0: long_amount[s] = 0.0 long_cost[s] = 0.0 past_rolling_long_entries[s] = [] if shrt_cost[s] > min_exit_cost: shrt_exit_price_list[s].append({'price': exit_bid[s], 'timestamp': row.Index}) if getattr(row, lows[s]) < exit_bid[s]: # shrt buy shrt_buy_cost = shrt_amount[s] * exit_bid[s] if balance[quot] >= shrt_buy_cost: # shrt buy normal shrt_buy_cost = max(shrt_buy_cost, min(balance[quot], -balance[s2c[s]] * exit_bid[s])) shrt_buy_amount = shrt_buy_cost / exit_bid[s] shrt_exits[s].append({'price': exit_bid[s], 'amount': shrt_buy_amount, 'timestamp': row.Index}) balance[quot] -= shrt_buy_cost balance[s2c[s]] += shrt_buy_amount * fee shrt_amount[s] = 0.0 shrt_cost[s] = 0.0 else: # partial shrt buy quot_avbl = max(0.0, balance[quot]) to_borrow = min(credit_avbl_quot, shrt_buy_cost - quot_avbl) partial_sel_cost = quot_avbl + to_borrow if partial_sel_cost > 0.0: coin_acquired = partial_sel_cost / exit_bid[s] shrt_exits[s].append({'price': exit_bid[s], 'amount': coin_acquired, 'timestamp': row.Index}) credit_avbl_quot -= to_borrow balance[quot] -= partial_sel_cost balance[s2c[s]] += coin_acquired * fee shrt_amount[s] -= coin_acquired shrt_cost[s] -= partial_sel_cost if shrt_amount[s] <= 0.0 or shrt_cost[s] <= 0.0: shrt_amount[s] = 0.0 shrt_cost[s] = 0.0 past_rolling_shrt_entries[s] = [] entry_bid[s] = round_dn( min(getattr(row, means[s]), getattr(row, min_emas[s])), price_precisions[s]) entry_ask[s] = round_up( max(getattr(row, means[s]), getattr(row, max_emas[s])), price_precisions[s]) acc_equity_quot = \ balance[quot] + sum([balance[s2c[s]] * getattr(row, means[s]) for s in symbols]) balance_list.append({**{s2c[s]: balance[s2c[s]] * getattr(row, means[s]) for s in symbols}, **{'acc_equity_quot': acc_equity_quot, 'timestamp': row.Index, quot: balance[quot]}}) acc_debt_quot = -sum([balance_list[-1][c] for c in balance if balance_list[-1][c] < 0.0]) balance_list[-1]['acc_debt_quot'] = acc_debt_quot if row.Index % 86400000 == 0 or row.Index >= end_ts: n_millis = row.Index - start_ts line = f'\r{(n_millis / ts_range) * 100:.2f}% ' line += f'acc equity quot: {acc_equity_quot:.6f} ' n_days = n_millis / 1000 / 60 / 60 / 24 line += f'avg daily gain: {acc_equity_quot**(1/n_days):6f} ' line += f'cost {cost:.8f} ' sys.stdout.write(line) sys.stdout.flush() return balance_list, long_entries, shrt_entries, long_exits, shrt_exits, \ long_exit_price_list, shrt_exit_price_list
def update_ideal_orders(self, s: str): coin, quot = self.symbol_split[s] other_bids = calc_other_orders(self.my_bids[s], self.cm.order_book[s]['bids']) other_asks = calc_other_orders(self.my_asks[s], self.cm.order_book[s]['asks']) highest_other_bid = sorted(other_bids, key=lambda x: x['price'])[-1] if other_bids else \ self.my_bids[s][-1] lowest_other_ask = sorted(other_asks, key=lambda x: x['price'])[0] if other_asks else \ self.my_asks[s][0] other_bid_incr = round( highest_other_bid['price'] + 10**-self.price_precisions[s], self.price_precisions[s]) other_ask_decr = round( lowest_other_ask['price'] - 10**-self.price_precisions[s], self.price_precisions[s]) small_trade_cost_default = max( self.min_trade_costs[s], (self.balance[quot]['account_equity'] * self.hyperparams['account_equity_pct_per_trade'])) small_trade_cost = max( 10**-self.amount_precisions[s] * self.cm.last_price[s], small_trade_cost_default) approx_small_trade_amount = round_up( small_trade_cost / self.cm.last_price[s], self.amount_precisions[s]) min_big_trade_amount = approx_small_trade_amount * 6 long_cost_vol, shrt_cost_vol = calc_rolling_cost_vol( self.my_trades[s], self.my_trades_analyses[s]['small_big_amount_threshold'], self.cc.milliseconds() - self.hyperparams['millis_rolling_small_trade_window']) all_shrt_buys = [] for s_ in self.symbols: price_ = self.my_trades_analyses[s_]['true_shrt_vwap'] * \ self.hyperparams['profit_pct_minus'] c_, q_ = self.symbol_split[s_] amount_ = self.balance[c_]['borrowed'] cost_ = amount_ * price_ all_shrt_buys.append({ 'price': price_, 'amount': amount_, 'cost': cost_, 'symbol': s_ }) quot_locked_in_shrt_buys = sum([e['cost'] for e in all_shrt_buys]) quot_locked_in_long_buys = \ sum([self.ideal_long_buy[s_]['amount'] * self.ideal_long_buy[s_]['price'] for s_ in self.symbols]) self.quot_locked_in_shrt_buys = quot_locked_in_shrt_buys # small orders # # long_buy # if s in self.do_long_buy: long_buy_cost = min([ small_trade_cost, self.balance[quot]['onhand'] / len(self.symbols), (self.balance[quot]['account_equity'] * self.hyperparams['account_equity_pct_per_period'] - long_cost_vol) ]) long_buy_cost = long_buy_cost if long_buy_cost > self.min_trade_costs[ s] else 0.0 long_buy_price = min([ round_dn(self.cm.min_ema[s], self.price_precisions[s]), other_ask_decr, (other_bid_incr if long_buy_cost / other_bid_incr < highest_other_bid['amount'] else highest_other_bid['price']) ]) self.ideal_long_buy[s] = { 'side': 'buy', 'amount': round_up(long_buy_cost / long_buy_price, self.amount_precisions[s]), 'price': long_buy_price } else: self.ideal_long_buy[s] = { 'side': 'buy', 'amount': 0.0, 'price': 0.0 } ############ # shrt_sel # if s in self.do_shrt_sel: shrt_sel_cost = min([ small_trade_cost, self.tradable_bnb if coin == 'BNB' else self.balance[coin]['onhand'], (self.balance[quot]['account_equity'] * self.hyperparams['account_equity_pct_per_period'] - shrt_cost_vol) ]) shrt_sel_cost = shrt_sel_cost if shrt_sel_cost > self.min_trade_costs[ s] else 0.0 shrt_sel_price = max([ round_up(self.cm.max_ema[s], self.price_precisions[s]), other_bid_incr, (other_ask_decr if shrt_sel_cost / other_ask_decr < lowest_other_ask['amount'] else lowest_other_ask['price']) ]) shrt_sel_amount = min( round_dn( self.balance[coin]['onhand'] + self.ideal_borrow[coin], self.amount_precisions[s]), round_up(shrt_sel_cost / shrt_sel_price, self.amount_precisions[s])) if shrt_sel_amount > self.min_trade_costs[s] / shrt_sel_price: self.ideal_shrt_sel[s] = { 'side': 'sell', 'amount': shrt_sel_amount, 'price': shrt_sel_price } else: self.ideal_shrt_sel[s] = { 'side': 'sell', 'amount': 0.0, 'price': 0.0 } else: self.ideal_shrt_sel[s] = { 'side': 'sell', 'amount': 0.0, 'price': 0.0 } ########### # debt adjustments # ideal_quot_onhand = quot_locked_in_shrt_buys + quot_locked_in_long_buys self.ideal_borrow[quot] = max( 0.0, min(ideal_quot_onhand - self.balance[quot]['onhand'], self.balance[quot]['borrowable'])) ideal_repay_quot = max( 0.0, min([(self.balance[quot]['borrowed'] + self.balance[quot]['interest']), self.balance[quot]['free'], self.balance[quot]['onhand'] - ideal_quot_onhand])) self.ideal_repay[quot] = ideal_repay_quot \ if ideal_repay_quot > quot_locked_in_long_buys else 0.0 #------------------# ideal_coin_debt = self.my_trades_analyses[s]['true_shrt_amount'] + \ self.ideal_shrt_sel[s]['amount'] self.ideal_borrow[coin] = max( 0.0, min(self.balance[coin]['borrowable'], ideal_coin_debt - self.balance[coin]['onhand'])) ideal_repay_coin = min([ self.balance[coin]['free'], self.balance[coin]['debt'], self.balance[coin]['onhand'] - ideal_coin_debt ]) self.ideal_repay[coin] = ideal_repay_coin \ if ideal_repay_coin > approx_small_trade_amount * 2 else 0.0 #################### # big orders # # long_sel # if s in self.do_long_sel: long_sel_amount = ((self.tradable_bnb if coin == 'BNB' else self.balance[coin]['onhand']) - self.ideal_shrt_sel[s]['amount'] - self.ideal_repay[coin] + self.ideal_borrow[coin]) long_sel_amount = max( 0.0, round_dn(long_sel_amount, self.amount_precisions[s])) if long_sel_amount > min_big_trade_amount: long_sel_price = max([ round_up(self.cm.max_ema[s], self.price_precisions[s]), other_bid_incr, (other_ask_decr if long_sel_amount < lowest_other_ask['amount'] else lowest_other_ask['price']), round_up((self.my_trades_analyses[s]['true_long_vwap'] * self.hyperparams['profit_pct_plus']), self.price_precisions[s]) ]) else: long_sel_price = 0.0 self.ideal_long_sel[s] = { 'side': 'sell', 'amount': long_sel_amount, 'price': long_sel_price } else: self.ideal_long_sel[s] = { 'side': 'sell', 'amount': 0.0, 'price': 0.0 } ############ # shrt_buy # if s in self.do_shrt_buy: shrt_buy_amount = (self.balance[coin]['borrowed'] + self.ideal_borrow[coin] - self.ideal_repay[coin] - self.ideal_shrt_sel[s]['amount']) shrt_buy_amount = max(0.0, round_dn(shrt_buy_amount)) shrt_buy_price = min([ round_dn(self.cm.min_ema[s], self.price_precisions[s]), other_ask_decr, (other_bid_incr if shrt_buy_amount < highest_other_bid['amount'] else highest_other_bid['price']), round_dn((self.my_trades_analyses[s]['true_shrt_vwap'] * self.hyperparams['profit_pct_minus']), self.price_precisions[s]), ]) if shrt_buy_amount > min_big_trade_amount: if self.balance[quot]['onhand'] + self.ideal_borrow[ quot] < ideal_quot_onhand: # means we are max leveraged shrt_buys_sorted_by_price_diff = sorted( [ e for e in all_shrt_buys if e['cost'] > small_trade_cost_default * 6 ], key=lambda x: self.cm.last_price[x['symbol']] / x[ 'price']) tmp_sum = quot_locked_in_long_buys eligible_shrt_buys = [] for sb in shrt_buys_sorted_by_price_diff: tmp_sum += sb['cost'] if tmp_sum >= self.balance[quot]['onhand']: break eligible_shrt_buys.append(sb) if s not in set( map(lambda x: x['symbol'], eligible_shrt_buys)): shrt_buy_price = 0.0 else: shrt_buy_price = 0.0 self.ideal_shrt_buy[s] = { 'side': 'buy', 'amount': shrt_buy_amount, 'price': shrt_buy_price } else: self.ideal_shrt_buy[s] = { 'side': 'buy', 'amount': 0.0, 'price': 0.0 }
def backtest(df: pd.DataFrame, settings: dict): symbols = list(df.symbol.unique()) quot = symbols[0].split('/')[1] df_mid = df.iloc[int(len(df) * 0.5):int(len(df) * 0.55)] precisions = { s: find_precision(df_mid[df_mid.symbol == s].price.values) for s in symbols } ppctminus = { f'{c}/{quot}': 1 - settings['coins'][c]['profit_pct'] for c in settings['coins'] } ppctplus = { f'{c}/{quot}': 1 + settings['coins'][c]['profit_pct'] for c in settings['coins'] } s2c = {s: s.split('/')[0] for s in symbols} balance = {s2c[s]: 0.0 for s in s2c} balance[quot] = settings['start_quot'] balance_ito_quot = {c: balance[c] for c in balance} acc_equity_quot = settings['start_quot'] acc_debt_quot = 0.0 entries_list = [] exits_list = [] exit_prices_list = [] prev_long_entry_ts = {s: 0 for s in symbols} prev_shrt_entry_ts = {s: 0 for s in symbols} long_cost = {s: 0.0 for s in symbols} long_amount = {s: 0.0 for s in symbols} shrt_cost = {s: 0.0 for s in symbols} shrt_amount = {s: 0.0 for s in symbols} fee = 1 - settings['fee'] margin = settings['margin'] - 1 exponent = settings['entry_vol_modifier_exponent'] max_multiplier = settings['min_exit_cost_multiplier'] / 2 account_equity_pct_per_symbol_per_hour = { c: settings['coins'][c]['account_equity_pct_per_hour'] for c in settings['coins'] } millis_wait_until_next_long_entry = {s: 0 for s in symbols} millis_wait_until_next_shrt_entry = {s: 0 for s in symbols} coins_long = set( [c for c in settings['coins'] if settings['coins'][c]['long']]) coins_shrt = set( [c for c in settings['coins'] if settings['coins'][c]['shrt']]) balance_list = [] last_price = {s: 0.0 for s in s2c} start_ts, end_ts = df.index[0], df.index[-1] ts_range = end_ts - start_ts k = 0 hour_to_millis = 60 * 60 * 1000 for row in df.itertuples(): s = row.symbol last_price[s] = row.price coin = s2c[s] default_cost = max( acc_equity_quot * settings['coins'][coin]['account_equity_pct_per_trade'], settings['min_quot_cost']) credit_avbl_quot = max(0.0, acc_equity_quot * margin - acc_debt_quot) if row.entry: if row.is_buyer_maker: if coin in coins_long and \ row.Index - prev_long_entry_ts[s] >= millis_wait_until_next_long_entry[s]: cost = default_cost if long_cost[s] > 0.0: long_exit_price = \ long_cost[s] / long_amount[s] if long_amount[s] > 0.0 else row.price cost *= max( 1.0, min(max_multiplier, (long_exit_price / row.price)**exponent)) quot_avbl = max(0.0, balance[quot]) borrow_amount = max( 0.0, min(cost - quot_avbl, credit_avbl_quot)) cost = min(quot_avbl + borrow_amount, cost) if cost >= settings['min_quot_cost']: credit_avbl_quot -= borrow_amount amount = cost / row.entry_price balance[quot] -= cost balance[coin] += amount * fee entries_list.append({ 'symbol': s, 'timestamp': row.Index, 'side': 'buy', 'amount': amount, 'price': row.entry_price, 'cost': cost }) long_cost[s] += cost long_amount[s] += amount prev_long_entry_ts[s] = row.Index millis_wait_until_next_long_entry[s] = (default_cost * hour_to_millis) / \ (account_equity_pct_per_symbol_per_hour[coin] * acc_equity_quot) else: if coin in coins_shrt and \ row.Index - prev_shrt_entry_ts[s] >= millis_wait_until_next_shrt_entry[s]: cost = default_cost if shrt_cost[s] > 0.0: shrt_exit_price = \ shrt_cost[s] / shrt_amount[s] if shrt_amount[s] > 0.0 else row.price cost *= max( 1.0, min(max_multiplier, (row.price / shrt_exit_price)**exponent)) coin_avbl_quot = max(0.0, balance[coin]) * row.entry_price borrow_amount_quot = max( 0.0, min(cost - coin_avbl_quot, credit_avbl_quot)) cost = min(coin_avbl_quot + borrow_amount_quot, cost) if cost >= settings['min_quot_cost']: credit_avbl_quot -= borrow_amount_quot amount = cost / row.entry_price balance[coin] -= amount balance[quot] += cost * fee entries_list.append({ 'symbol': s, 'timestamp': row.Index, 'side': 'sell', 'amount': amount, 'price': row.entry_price, 'cost': cost }) shrt_cost[s] += cost shrt_amount[s] += amount prev_shrt_entry_ts[s] = row.Index millis_wait_until_next_shrt_entry[s] = (default_cost * hour_to_millis) / \ (account_equity_pct_per_symbol_per_hour[coin] * acc_equity_quot) bag_ratio = ( (long_amount[s] - shrt_amount[s]) * row.price) / acc_equity_quot if row.is_buyer_maker: min_exit_cost = default_cost * settings['min_exit_cost_multiplier'] try: shrt_vwap = shrt_cost[s] / shrt_amount[s] except ZeroDivisionError: shrt_vwap = 10e10 profit_pct = 1 - min( 0.1, max(settings['coins'][coin]['profit_pct'], bag_ratio * settings['profit_pct_multiplier'])) exit_price = min(row.exit_price, round_dn(shrt_vwap * profit_pct, precisions[s])) exit_cost = shrt_amount[s] * exit_price exit_prices_list.append({ 'timestamp': row.Index, 'symbol': s, 'side': 'buy', 'price': exit_price }) if coin in coins_shrt and shrt_amount[ s] * exit_price >= min_exit_cost: if row.price < exit_price and exit_cost >= min_exit_cost: quot_avbl = max(0.0, balance[quot]) borrow_amount = max( 0.0, min(exit_cost - quot_avbl, credit_avbl_quot)) exit_cost = min(quot_avbl + borrow_amount, exit_cost) if exit_cost >= min_exit_cost: balance[quot] -= exit_cost exit_amount = exit_cost / exit_price balance[coin] += exit_amount * fee exits_list.append({ 'symbol': s, 'timestamp': row.Index, 'side': 'buy', 'amount': exit_amount, 'price': exit_price, 'cost': exit_cost }) shrt_amount[s] -= exit_amount shrt_cost[s] -= exit_cost if shrt_amount[s] <= 0.0 or shrt_cost[s] <= 0.0: shrt_amount[s], shrt_cost[s] = 0.0, 0.0 else: min_exit_cost = default_cost * settings['min_exit_cost_multiplier'] try: long_vwap = long_cost[s] / max(long_amount[s], 9e-9) except ZeroDivisionError: long_vwap = 0.0 profit_pct = 1 + min( 0.1, max(settings['coins'][coin]['profit_pct'], -bag_ratio * settings['profit_pct_multiplier'])) exit_price = max(row.exit_price, round_up(long_vwap * profit_pct, precisions[s])) exit_prices_list.append({ 'timestamp': row.Index, 'symbol': s, 'side': 'sell', 'price': exit_price }) if coin in coins_long and long_amount[ s] * row.exit_price >= min_exit_cost: exit_cost = long_amount[s] * exit_price if coin in coins_long and row.price > exit_price and exit_cost >= min_exit_cost: coin_avbl_quot = max(0.0, balance[coin]) * exit_price borrow_amount_quot = max( 0.0, min(exit_cost - coin_avbl_quot, credit_avbl_quot)) exit_cost = min(coin_avbl_quot + borrow_amount_quot, exit_cost) if exit_cost >= min_exit_cost: exit_amount = exit_cost / exit_price balance[coin] -= exit_amount balance[quot] += exit_cost * fee exits_list.append({ 'symbol': s, 'timestamp': row.Index, 'side': 'sell', 'amount': exit_amount, 'price': exit_price, 'cost': exit_cost }) long_amount[s] -= exit_amount long_cost[s] -= exit_cost if long_amount[s] <= 0.0 or long_cost[s] <= 0.0: long_amount[s], long_cost[s] = 0.0, 0.0 acc_equity_quot -= (balance_ito_quot[coin] + balance_ito_quot[quot]) acc_debt_quot -= -(min(0.0, balance_ito_quot[coin]) + min(0.0, balance_ito_quot[quot])) balance_ito_quot[coin] = balance[coin] * row.price balance_ito_quot[quot] = balance[quot] acc_equity_quot += (balance_ito_quot[coin] + balance_ito_quot[quot]) acc_debt_quot += -(min(0.0, balance_ito_quot[coin]) + min(0.0, balance_ito_quot[quot])) onhand_ito_quot = sum([max(0.0, v) for v in balance_ito_quot.values()]) margin_level = onhand_ito_quot / acc_debt_quot if acc_debt_quot > 0.0 else 9e9 if margin_level <= settings['liquidation_margin_level']: print('\nliquidation!') return balance_list, entries_list, exits_list, exit_prices_list k += 1 if k % 5000 == 0: acc_equity_quot = sum(balance_ito_quot.values()) acc_debt_quot = -sum( [min(0.0, v) for v in balance_ito_quot.values()]) n_millis = row.Index - start_ts n_days = n_millis / 1000 / 60 / 60 / 24 avg_daily_gain = (acc_equity_quot / settings['start_quot'])**(1 / n_days) bag_ratios = { f'bag_ratio_{s2c[s]}': ((long_amount[s] - shrt_amount[s]) * last_price[s]) / acc_equity_quot for s in s2c } balance_list.append({ **balance_ito_quot, **{ 'acc_equity_quot': acc_equity_quot, 'acc_debt_quot': acc_debt_quot, 'onhand_ito_quot': onhand_ito_quot, 'credit_avbl_quot': credit_avbl_quot, 'avg_daily_gain': avg_daily_gain, 'timestamp': row.Index }, **bag_ratios }) line = f'\r{(n_millis / ts_range) * 100:.2f}% ' line += f'n_days {n_days:.2f} ' line += f'acc equity quot: {acc_equity_quot:.6f} ' line += f"avg daily gain: {avg_daily_gain:6f} " line += f'cost {default_cost:.8f} ' line += f'credit_avbl_quot {credit_avbl_quot:.6f} ' line += f'margin_level {margin_level:.4f} ' sys.stdout.write(line) sys.stdout.flush() return balance_list, entries_list, exits_list, exit_prices_list
def backtest(df: pd.DataFrame, settings: dict): symbols = settings['symbols'] precisions = settings['precisions'] s2c = {s: s.split('/')[0] for s in symbols} coins = sorted(set(s2c.values())) quot = symbols[0].split('/')[1] assert all([s.split('/')[1] == quot for s in symbols]) # max percentage of total account equity same side entry volume per symbol per hour account_equity_pct_per_hour = settings[ 'max_entry_acc_val_pct_per_hour'] / len(symbols) # max percentage of total account equity same side entry cost per symbol per entry account_equity_pct_per_entry = account_equity_pct_per_hour * settings[ 'min_entry_delay_hours'] entry_delay_millis = settings['min_entry_delay_hours'] * HOUR_TO_MILLIS print('account_equity_pct_per_entry', account_equity_pct_per_entry) print('account_equity_pct_per_hour', account_equity_pct_per_hour) print('min_entry_delay_hours', settings['min_entry_delay_hours']) margin_multiplier = settings['max_leverage'] - 1 exponent = settings['entry_vol_modifier_exponent'] min_exit_cost_multiplier = settings['min_exit_cost_multiplier'] n_days_to_min_markup = settings['n_days_to_min_markup'] fee = 0.999 equity = {coin: 0.0 for coin in coins} equity[quot] = 1.0 equity_ito_quot = equity.copy() account_equity = equity[quot] debt = 0.0 debt_neg = 0.0 onhand = equity[quot] long_cost = {s: 0.0 for s in symbols} long_amount = {s: 0.0 for s in symbols} shrt_cost = {s: 0.0 for s in symbols} shrt_amount = {s: 0.0 for s in symbols} logs = [] trades = {s: [] for s in symbols} prev_entry_ts = { 'long': {s: 0 for s in symbols}, 'shrt': {s: 0 for s in symbols} } prev_exit_ts = { 'long': {s: df.index[0] for s in symbols}, 'shrt': {s: df.index[0] for s in symbols} } last_price = {s: 0.0 for s in symbols} for s in symbols: for row in df.itertuples(): if row.symbol == s: last_price[s] = row.avg break last_price[f'{quot}/{quot}'] = 1.0 start_ts, end_ts = df.index[0], df.index[-1] ts_range = end_ts - start_ts k = 0 kn = len(df) // 2000 liquidation = False for row in df.itertuples(): s = row.symbol coin = s2c[s] entry_cost = account_equity * account_equity_pct_per_entry credit = account_equity * margin_multiplier - debt_neg try: long_vwap = long_cost[s] / long_amount[s] except ZeroDivisionError: long_vwap = row.avg try: shrt_vwap = shrt_cost[s] / shrt_amount[s] except ZeroDivisionError: shrt_vwap = row.avg ##### long exit ##### long_bag_duration_days = (row.Index - prev_exit_ts['long'][s]) / DAY_TO_MILLIS long_exit_markup = max( settings['min_markup_pct'], settings['max_markup_pct'] * (1 - (long_bag_duration_days / n_days_to_min_markup))) long_exit_price = max( round_up(long_vwap * (1 + long_exit_markup), precisions[s]), row.exit_ask) if settings['long'] and row.high > long_exit_price: coin_avbl = credit / row.avg + max(0.0, equity[coin]) long_exit_amount = min(coin_avbl, long_amount[s]) long_exit_cost = long_exit_amount * long_exit_price if long_exit_cost > entry_cost * min_exit_cost_multiplier: equity[coin] -= long_exit_amount equity[quot] += long_exit_cost * fee trades[s].append({ 'timestamp': row.Index, 'side': 'sel', 'type': 'exit', 'price': long_exit_price, 'amount': long_exit_amount, 'cost': long_exit_cost, 'fee': long_exit_cost * (fee - 1) * -1 }) if long_exit_amount < long_amount[s]: # partial exit long_cost[s] -= long_exit_cost long_amount[s] -= long_exit_amount else: prev_exit_ts['long'][s] = row.Index long_cost[s] = 0.0 long_amount[s] = 0.0 ##### shrt exit ##### shrt_bag_duration_days = (row.Index - prev_exit_ts['shrt'][s]) / DAY_TO_MILLIS shrt_exit_markup = max( settings['min_markup_pct'], settings['max_markup_pct'] * (1 - (shrt_bag_duration_days / n_days_to_min_markup))) shrt_exit_price = min( round_dn(shrt_vwap * (1 - shrt_exit_markup), precisions[s]), row.exit_bid) if settings['shrt'] and row.low < shrt_exit_price: quot_avbl = credit + max(0.0, equity[quot]) shrt_exit_amount = min(quot_avbl / shrt_exit_price, shrt_amount[s]) shrt_exit_cost = shrt_exit_amount * shrt_exit_price if shrt_exit_cost > entry_cost * min_exit_cost_multiplier: equity[quot] -= shrt_exit_cost equity[coin] += shrt_amount[s] * fee trades[s].append({ 'timestamp': row.Index, 'side': 'buy', 'type': 'exit', 'price': shrt_exit_price, 'amount': shrt_amount[s], 'cost': shrt_exit_cost, 'fee': shrt_exit_cost * (fee - 1) * -1 }) if shrt_exit_amount < shrt_amount[s]: # partial exit shrt_cost[s] -= shrt_exit_cost shrt_amount[s] -= shrt_exit_amount else: prev_exit_ts['shrt'][s] = row.Index shrt_cost[s] = 0.0 shrt_amount[s] = 0.0 ##### long entry ##### if settings['long'] and row.Index - prev_entry_ts['long'][ s] >= entry_delay_millis: if row.low < row.entry_bid: long_entry_cost = entry_cost * max( 1.0, min(min_exit_cost_multiplier / 2, (long_vwap / row.entry_bid)**exponent)) if credit + max(0.0, equity[quot]) >= long_entry_cost: long_entry_amount = long_entry_cost / row.entry_bid equity[quot] -= long_entry_cost equity[coin] += long_entry_amount * fee long_cost[s] += long_entry_cost long_amount[s] += long_entry_amount prev_entry_ts['long'][s] = row.Index trades[s].append({ 'timestamp': row.Index, 'side': 'buy', 'type': 'entry', 'price': row.entry_bid, 'long_vwap': long_cost[s] / long_amount[s], 'amount': long_entry_amount, 'cost': long_entry_cost, 'fee': long_entry_cost * (fee - 1) * -1 }) ##### shrt entry ##### if settings['shrt'] and row.Index - prev_entry_ts['shrt'][ s] >= entry_delay_millis: if row.high > row.entry_ask: shrt_entry_cost = entry_cost * max( 1.0, min(min_exit_cost_multiplier / 2, (row.entry_ask / shrt_vwap)**exponent)) if credit + max( 0.0, equity[coin] * row.entry_ask) >= shrt_entry_cost: shrt_entry_amount = shrt_entry_cost / row.entry_ask equity[coin] -= shrt_entry_amount equity[quot] += shrt_entry_cost * fee shrt_cost[s] += shrt_entry_cost shrt_amount[s] += shrt_entry_amount prev_entry_ts['shrt'][s] = row.Index trades[s].append({ 'timestamp': row.Index, 'side': 'sel', 'type': 'entry', 'price': row.entry_ask, 'shrt_vwap': shrt_cost[s] / shrt_amount[s], 'amount': shrt_entry_amount, 'cost': shrt_entry_cost, 'fee': shrt_entry_cost * (fee - 1) * -1 }) account_equity -= (equity_ito_quot[coin] + equity_ito_quot[quot]) debt -= (min(0.0, equity_ito_quot[coin]) + min(0.0, equity_ito_quot[quot])) onhand -= (max(0.0, equity_ito_quot[coin]) + max(0.0, equity_ito_quot[quot])) equity_ito_quot[s2c[s]] = equity[s2c[s]] * row.avg equity_ito_quot[quot] = equity[quot] account_equity += (equity_ito_quot[coin] + equity_ito_quot[quot]) debt += (min(0.0, equity_ito_quot[coin]) + min(0.0, equity_ito_quot[quot])) onhand += (max(0.0, equity_ito_quot[coin]) + max(0.0, equity_ito_quot[quot])) debt_neg = round(max(0.0, abs(debt)), 4) margin_level = min(5.0, onhand / debt_neg if debt_neg else 5.0) if margin_level < 1.05: print('\nliquidation!') print(debt) print(equity_ito_quot) print(equity) print(row) liquidation = True k = kn - 1 k += 1 if k % kn == 0: log_entry = { **{ 'timestamp': row.Index, 'debt': debt_neg, 'onhand': onhand, 'credit': credit, 'margin_level': margin_level, 'equity': account_equity }, **equity_ito_quot } logs.append(log_entry) n_millis = row.Index - start_ts n_days = n_millis / (1000 * 60 * 60 * 24) adg = account_equity**(1 / n_days) ayg = adg**365 line = f'{n_millis / ts_range:.2f} margin_level, {margin_level:.2f}' line += f' equity {account_equity:.4f} credit {credit:4f} ayg {ayg:.6f}' sys.stdout.write('\r' + line + ' ' * 8) if liquidation: break return logs, trades