def prep_df(ohlcvs: pd.DataFrame, settings: dict) -> pd.DataFrame:
ema_spans = settings['ema_spans_minutes']
entry_spread = settings['entry_spread']
precision = calc_price_precision(ohlcvs.close)
emas = pd.concat([
pd.Series(ohlcvs.close.ewm(span=span, adjust=False).mean(),
name=str(span)) for span in ema_spans
],
axis=1)
min_ema = emas.min(axis=1)
max_ema = emas.max(axis=1)
entry_bid = round_dn(min_ema * (1 - entry_spread), precision)
entry_ask = round_up(max_ema * (1 + entry_spread), precision)
exit_bid = round_dn(min_ema, precision)
exit_ask = round_up(max_ema, precision)
avg = ohlcvs[['open', 'close']].mean(axis=1)
df = pd.DataFrame(
{
'entry_bid': entry_bid,
'entry_ask': entry_ask,
'exit_bid': exit_bid,
'exit_ask': exit_ask,
'avg': avg,
'high': ohlcvs.high,
'low': ohlcvs.low
},
index=ohlcvs.index)
return df[(df.low < df.exit_bid) | (df.high > df.exit_ask)]
def create_df(symbols: [str],
n_days: int,
settings: dict,
no_download: bool = False):
dfs = []
for s in symbols:
coin, quot = s.split('/')
print('preparing', s)
rt = trade_data.fetch_raw_trades(s,
n_days=n_days,
no_download=no_download)
print('precision')
precision = find_precision(rt.price)
print('emas')
start_ts = time()
rt_emas = calc_ema_from_raw_trades(rt, settings['ema_spans_minutes'])
print('elapsed seconds calc emas', round(time() - start_ts, 2))
ema_min = rt_emas.min(axis=1)
ema_max = rt_emas.max(axis=1)
long_entry = round_dn(
ema_min * (1 - settings['coins'][coin]['entry_spread']), precision)
shrt_entry = round_up(
ema_max * (1 + settings['coins'][coin]['entry_spread']), precision)
long_exit = round_up(ema_max, precision)
shrt_exit = round_dn(ema_min, precision)
rt.loc[:, 'entry_price'] = long_entry.where(rt.is_buyer_maker,
shrt_entry)
rt.loc[:, 'exit_price'] = long_exit.where(~rt.is_buyer_maker,
shrt_exit)
rt.loc[:, 'entry'] = ((rt.is_buyer_maker & (rt.price < long_entry)) |
(~rt.is_buyer_maker & (rt.price > shrt_entry)))
rt = rt[((rt.is_buyer_maker & (rt.price < shrt_exit)) |
(~rt.is_buyer_maker & (rt.price > long_exit)))]
rt.loc[:, 'symbol'] = np.repeat(s, len(rt))
dfs.append(rt.set_index('timestamp'))
return pd.concat(dfs, axis=0).sort_index()
def backtest(df: pd.DataFrame, settings: dict, price_precisions: dict = {}):
start_quot = 1.0
ppctminus = 1 - settings['profit_pct']
ppctplus = 1 + settings['profit_pct']
symbols = [c.replace('_low', '') for c in df.columns if 'low' in c]
if not price_precisions:
price_precisions = {s: 8 for s in symbols}
lows = {s: f'{s}_low' for s in symbols}
highs = {s: f'{s}_high' for s in symbols}
means = {s: f'{s}_mean' for s in symbols}
min_emas = {s: f'{s}_mean_min_ema' for s in symbols}
max_emas = {s: f'{s}_mean_max_ema' for s in symbols}
min_delay_millis = settings['min_seconds_between_same_side_entries'] * 1000
rolling_millis = settings['max_memory_span_days'] * 24 * 60 * 60 * 1000
s2c = {s: s.split('_')[0] for s in symbols}
quot = symbols[0].split('_')[1]
balance = {s2c[s]: 0.0 for s in s2c}
balance[quot] = 1.0
acc_equity_quot = 1.0
acc_debt_quot = 0.0
long_entries = {s: [] for s in symbols}
shrt_entries = {s: [] for s in symbols}
long_exits = {s: [] for s in symbols}
shrt_exits = {s: [] for s in symbols}
long_exit_price_list = {s: [] for s in symbols}
shrt_exit_price_list = {s: [] for s in symbols}
past_rolling_long_entries = {s: [] for s in symbols}
past_rolling_shrt_entries = {s: [] for s in symbols}
entry_bid = {s: round(df.iloc[0][means[s]], 8) for s in symbols}
entry_ask = {s: round(df.iloc[0][means[s]], 8) for s in symbols}
exit_bid = {s: entry_bid[s] for s in symbols}
exit_ask = {s: entry_ask[s] for s in symbols}
long_cost = {s: 0.0 for s in symbols}
long_amount = {s: 0.0 for s in symbols}
shrt_cost = {s: 0.0 for s in symbols}
shrt_amount = {s: 0.0 for s in symbols}
fee = 1 - 0.000675 # vip 1
margin_level = 3 - 1
balance_list = []
do_shrt = {s for s in symbols if s2c[s] in settings['coins_shrt']}
do_long = {s for s in symbols if s2c[s] in settings['coins_long']}
exponent = settings['entry_vol_modifier_exponent']
start_ts, end_ts = df.index[0], df.index[-1]
ts_range = end_ts - start_ts
for row in df.itertuples():
cost = acc_equity_quot * settings['account_equity_pct_per_trade']
min_exit_cost = cost * settings['min_big_trade_cost_multiplier']
credit_avbl_quot = max(0.0, acc_equity_quot * margin_level - acc_debt_quot)
age_limit = row.Index - rolling_millis
for s in symbols:
# rolling longs
long_i = get_cutoff_index(past_rolling_long_entries[s], age_limit)
if long_i > 0:
slc = past_rolling_long_entries[s][:long_i]
past_rolling_long_entries[s] = past_rolling_long_entries[s][long_i:]
long_amount[s] -= sum([e['amount'] for e in slc])
long_cost[s] -= sum([e['amount'] * e['price'] for e in slc])
if long_cost[s] <= 0.0 or long_amount[s] <= 0.0:
long_cost[s] = 0.0
long_amount[s] = 0.0
past_rolling_long_entries[s] = []
exit_ask[s] = getattr(row, means[s])
else:
exit_ask[s] = (long_cost[s] / long_amount[s]) * ppctplus
# rolling shrts
shrt_i = get_cutoff_index(past_rolling_shrt_entries[s], age_limit)
if shrt_i > 0:
slc = past_rolling_shrt_entries[s][:shrt_i]
past_rolling_shrt_entries[s] = past_rolling_shrt_entries[s][shrt_i:]
shrt_cost[s] -= sum([e['amount'] * e['price'] for e in slc])
shrt_amount[s] -= sum([e['amount'] for e in slc])
if shrt_cost[s] <= 0.0 or shrt_amount[s] <= 0.0:
shrt_cost[s] = 0.0
shrt_amount[s] = 0.0
past_rolling_shrt_entries[s] = []
exit_bid[s] = getattr(row, means[s])
else:
exit_bid[s] = (shrt_cost[s] / shrt_amount[s]) * ppctminus
if s in do_long and getattr(row, lows[s]) < entry_bid[s] and \
(not long_entries[s] or
(row.Index - long_entries[s][-1]['timestamp'] >= min_delay_millis)):
# long buy
long_modifier = max(
1.0, min(settings['min_big_trade_cost_multiplier'] - 1,
(exit_ask[s] / getattr(row, means[s]))**exponent))
buy_cost = cost * long_modifier
if balance[quot] >= buy_cost:
# long buy normal
buy_amount = (buy_cost / entry_bid[s])
balance[quot] -= buy_cost
balance[s2c[s]] += buy_amount * fee
long_entries[s].append({'price': entry_bid[s], 'amount': buy_amount,
'timestamp': row.Index})
past_rolling_long_entries[s].append(long_entries[s][-1])
long_amount[s] += buy_amount
long_cost[s] += buy_cost
exit_ask[s] = (long_cost[s] / long_amount[s]) * ppctplus
elif credit_avbl_quot > 0.0:
# long buy with credit
quot_avbl = max(0.0, balance[quot])
to_borrow = min(credit_avbl_quot, buy_cost - quot_avbl)
credit_avbl_quot -= to_borrow
partial_buy_cost = quot_avbl + to_borrow
buy_amount = (partial_buy_cost / entry_bid[s])
balance[quot] -= partial_buy_cost
balance[s2c[s]] += buy_amount * fee
long_entries[s].append({'price': entry_bid[s], 'amount': buy_amount,
'timestamp': row.Index})
past_rolling_long_entries[s].append(long_entries[s][-1])
long_amount[s] += buy_amount
long_cost[s] += partial_buy_cost
exit_ask[s] = (long_cost[s] / long_amount[s]) * ppctplus
if s in do_shrt and getattr(row, highs[s]) > entry_ask[s] and \
(not shrt_entries[s] or
(row.Index - shrt_entries[s][-1]['timestamp'] >= min_delay_millis)):
# shrt sel
shrt_modifier = max(
1.0, min(settings['min_big_trade_cost_multiplier'] - 1,
(getattr(row, means[s]) / exit_bid[s])**exponent))
sel_cost = cost * shrt_modifier
sel_amount = sel_cost / entry_ask[s]
if balance[s2c[s]] >= sel_amount:
# shrt sel normal
balance[s2c[s]] -= sel_amount
balance[quot] += sel_cost * fee
shrt_entries[s].append({'price': entry_ask[s], 'amount': sel_amount,
'timestamp': row.Index})
past_rolling_shrt_entries[s].append(shrt_entries[s][-1])
shrt_amount[s] += sel_amount
shrt_cost[s] += sel_cost
exit_bid[s] = (shrt_cost[s] / shrt_amount[s]) * ppctminus
elif credit_avbl_quot > 0.0:
# shrt sel with credit
coin_avbl = max(0.0, balance[s2c[s]])
to_borrow = min(credit_avbl_quot / entry_ask[s], sel_amount - coin_avbl)
credit_avbl_quot -= (to_borrow * entry_ask[s])
partial_sel_amount = coin_avbl + to_borrow
balance[s2c[s]] -= partial_sel_amount
partial_sel_cost = partial_sel_amount * entry_ask[s]
balance[quot] += partial_sel_cost * fee
shrt_entries[s].append({'price': entry_ask[s], 'amount': partial_sel_amount,
'timestamp': row.Index})
past_rolling_shrt_entries[s].append(shrt_entries[s][-1])
shrt_amount[s] += partial_sel_amount
shrt_cost[s] += partial_sel_cost
exit_bid[s] = (shrt_cost[s] / shrt_amount[s]) * ppctminus
exit_ask[s] = round_up(exit_ask[s], price_precisions[s])
exit_bid[s] = round_dn(exit_bid[s], price_precisions[s])
if long_cost[s] > min_exit_cost:
# long sel
long_exit_price_list[s].append({'price': exit_ask[s], 'timestamp': row.Index})
if getattr(row, highs[s]) > exit_ask[s]:
if balance[s2c[s]] >= long_amount[s]:
# long sel normal
long_sel_amount = max(balance[s2c[s]], long_amount[s])
long_exits[s].append({'price': exit_ask[s], 'amount': long_sel_amount,
'timestamp': row.Index})
quot_acquired = long_sel_amount * exit_ask[s]
balance[s2c[s]] -= long_sel_amount
balance[quot] += quot_acquired * fee
long_amount[s] = 0.0
long_cost[s] = 0.0
else:
# partial long sel
coin_avbl = max(0.0, balance[s2c[s]])
to_borrow = min(credit_avbl_quot / exit_ask[s], long_amount[s] - coin_avbl)
partial_sel_amount = coin_avbl + to_borrow
if partial_sel_amount > 0.0:
credit_avbl_quot -= (to_borrow * exit_ask[s])
balance[s2c[s]] -= partial_sel_amount
partial_sel_cost = partial_sel_amount * exit_ask[s]
balance[quot] += partial_sel_cost * fee
long_exits[s].append({'price': exit_ask[s],
'amount': partial_sel_amount,
'timestamp': row.Index})
long_amount[s] -= partial_sel_amount
long_cost[s] -= partial_sel_cost
if long_amount[s] <= 0.0 or long_cost[s] <= 0.0:
long_amount[s] = 0.0
long_cost[s] = 0.0
past_rolling_long_entries[s] = []
if shrt_cost[s] > min_exit_cost:
shrt_exit_price_list[s].append({'price': exit_bid[s], 'timestamp': row.Index})
if getattr(row, lows[s]) < exit_bid[s]:
# shrt buy
shrt_buy_cost = shrt_amount[s] * exit_bid[s]
if balance[quot] >= shrt_buy_cost:
# shrt buy normal
shrt_buy_cost = max(shrt_buy_cost,
min(balance[quot], -balance[s2c[s]] * exit_bid[s]))
shrt_buy_amount = shrt_buy_cost / exit_bid[s]
shrt_exits[s].append({'price': exit_bid[s], 'amount': shrt_buy_amount,
'timestamp': row.Index})
balance[quot] -= shrt_buy_cost
balance[s2c[s]] += shrt_buy_amount * fee
shrt_amount[s] = 0.0
shrt_cost[s] = 0.0
else:
# partial shrt buy
quot_avbl = max(0.0, balance[quot])
to_borrow = min(credit_avbl_quot, shrt_buy_cost - quot_avbl)
partial_sel_cost = quot_avbl + to_borrow
if partial_sel_cost > 0.0:
coin_acquired = partial_sel_cost / exit_bid[s]
shrt_exits[s].append({'price': exit_bid[s], 'amount': coin_acquired,
'timestamp': row.Index})
credit_avbl_quot -= to_borrow
balance[quot] -= partial_sel_cost
balance[s2c[s]] += coin_acquired * fee
shrt_amount[s] -= coin_acquired
shrt_cost[s] -= partial_sel_cost
if shrt_amount[s] <= 0.0 or shrt_cost[s] <= 0.0:
shrt_amount[s] = 0.0
shrt_cost[s] = 0.0
past_rolling_shrt_entries[s] = []
entry_bid[s] = round_dn(
min(getattr(row, means[s]), getattr(row, min_emas[s])), price_precisions[s])
entry_ask[s] = round_up(
max(getattr(row, means[s]), getattr(row, max_emas[s])), price_precisions[s])
acc_equity_quot = \
balance[quot] + sum([balance[s2c[s]] * getattr(row, means[s]) for s in symbols])
balance_list.append({**{s2c[s]: balance[s2c[s]] * getattr(row, means[s]) for s in symbols},
**{'acc_equity_quot': acc_equity_quot, 'timestamp': row.Index,
quot: balance[quot]}})
acc_debt_quot = -sum([balance_list[-1][c] for c in balance if balance_list[-1][c] < 0.0])
balance_list[-1]['acc_debt_quot'] = acc_debt_quot
if row.Index % 86400000 == 0 or row.Index >= end_ts:
n_millis = row.Index - start_ts
line = f'\r{(n_millis / ts_range) * 100:.2f}% '
line += f'acc equity quot: {acc_equity_quot:.6f} '
n_days = n_millis / 1000 / 60 / 60 / 24
line += f'avg daily gain: {acc_equity_quot**(1/n_days):6f} '
line += f'cost {cost:.8f} '
sys.stdout.write(line)
sys.stdout.flush()
return balance_list, long_entries, shrt_entries, long_exits, shrt_exits, \
long_exit_price_list, shrt_exit_price_list
def update_ideal_orders(self, s: str):
coin, quot = self.symbol_split[s]
other_bids = calc_other_orders(self.my_bids[s],
self.cm.order_book[s]['bids'])
other_asks = calc_other_orders(self.my_asks[s],
self.cm.order_book[s]['asks'])
highest_other_bid = sorted(other_bids, key=lambda x: x['price'])[-1] if other_bids else \
self.my_bids[s][-1]
lowest_other_ask = sorted(other_asks, key=lambda x: x['price'])[0] if other_asks else \
self.my_asks[s][0]
other_bid_incr = round(
highest_other_bid['price'] + 10**-self.price_precisions[s],
self.price_precisions[s])
other_ask_decr = round(
lowest_other_ask['price'] - 10**-self.price_precisions[s],
self.price_precisions[s])
small_trade_cost_default = max(
self.min_trade_costs[s],
(self.balance[quot]['account_equity'] *
self.hyperparams['account_equity_pct_per_trade']))
small_trade_cost = max(
10**-self.amount_precisions[s] * self.cm.last_price[s],
small_trade_cost_default)
approx_small_trade_amount = round_up(
small_trade_cost / self.cm.last_price[s],
self.amount_precisions[s])
min_big_trade_amount = approx_small_trade_amount * 6
long_cost_vol, shrt_cost_vol = calc_rolling_cost_vol(
self.my_trades[s],
self.my_trades_analyses[s]['small_big_amount_threshold'],
self.cc.milliseconds() -
self.hyperparams['millis_rolling_small_trade_window'])
all_shrt_buys = []
for s_ in self.symbols:
price_ = self.my_trades_analyses[s_]['true_shrt_vwap'] * \
self.hyperparams['profit_pct_minus']
c_, q_ = self.symbol_split[s_]
amount_ = self.balance[c_]['borrowed']
cost_ = amount_ * price_
all_shrt_buys.append({
'price': price_,
'amount': amount_,
'cost': cost_,
'symbol': s_
})
quot_locked_in_shrt_buys = sum([e['cost'] for e in all_shrt_buys])
quot_locked_in_long_buys = \
sum([self.ideal_long_buy[s_]['amount'] * self.ideal_long_buy[s_]['price']
for s_ in self.symbols])
self.quot_locked_in_shrt_buys = quot_locked_in_shrt_buys
# small orders #
# long_buy #
if s in self.do_long_buy:
long_buy_cost = min([
small_trade_cost,
self.balance[quot]['onhand'] / len(self.symbols),
(self.balance[quot]['account_equity'] *
self.hyperparams['account_equity_pct_per_period'] -
long_cost_vol)
])
long_buy_cost = long_buy_cost if long_buy_cost > self.min_trade_costs[
s] else 0.0
long_buy_price = min([
round_dn(self.cm.min_ema[s], self.price_precisions[s]),
other_ask_decr,
(other_bid_incr if
long_buy_cost / other_bid_incr < highest_other_bid['amount']
else highest_other_bid['price'])
])
self.ideal_long_buy[s] = {
'side':
'buy',
'amount':
round_up(long_buy_cost / long_buy_price,
self.amount_precisions[s]),
'price':
long_buy_price
}
else:
self.ideal_long_buy[s] = {
'side': 'buy',
'amount': 0.0,
'price': 0.0
}
############
# shrt_sel #
if s in self.do_shrt_sel:
shrt_sel_cost = min([
small_trade_cost, self.tradable_bnb
if coin == 'BNB' else self.balance[coin]['onhand'],
(self.balance[quot]['account_equity'] *
self.hyperparams['account_equity_pct_per_period'] -
shrt_cost_vol)
])
shrt_sel_cost = shrt_sel_cost if shrt_sel_cost > self.min_trade_costs[
s] else 0.0
shrt_sel_price = max([
round_up(self.cm.max_ema[s], self.price_precisions[s]),
other_bid_incr,
(other_ask_decr
if shrt_sel_cost / other_ask_decr < lowest_other_ask['amount']
else lowest_other_ask['price'])
])
shrt_sel_amount = min(
round_dn(
self.balance[coin]['onhand'] + self.ideal_borrow[coin],
self.amount_precisions[s]),
round_up(shrt_sel_cost / shrt_sel_price,
self.amount_precisions[s]))
if shrt_sel_amount > self.min_trade_costs[s] / shrt_sel_price:
self.ideal_shrt_sel[s] = {
'side': 'sell',
'amount': shrt_sel_amount,
'price': shrt_sel_price
}
else:
self.ideal_shrt_sel[s] = {
'side': 'sell',
'amount': 0.0,
'price': 0.0
}
else:
self.ideal_shrt_sel[s] = {
'side': 'sell',
'amount': 0.0,
'price': 0.0
}
###########
# debt adjustments #
ideal_quot_onhand = quot_locked_in_shrt_buys + quot_locked_in_long_buys
self.ideal_borrow[quot] = max(
0.0,
min(ideal_quot_onhand - self.balance[quot]['onhand'],
self.balance[quot]['borrowable']))
ideal_repay_quot = max(
0.0,
min([(self.balance[quot]['borrowed'] +
self.balance[quot]['interest']), self.balance[quot]['free'],
self.balance[quot]['onhand'] - ideal_quot_onhand]))
self.ideal_repay[quot] = ideal_repay_quot \
if ideal_repay_quot > quot_locked_in_long_buys else 0.0
#------------------#
ideal_coin_debt = self.my_trades_analyses[s]['true_shrt_amount'] + \
self.ideal_shrt_sel[s]['amount']
self.ideal_borrow[coin] = max(
0.0,
min(self.balance[coin]['borrowable'],
ideal_coin_debt - self.balance[coin]['onhand']))
ideal_repay_coin = min([
self.balance[coin]['free'], self.balance[coin]['debt'],
self.balance[coin]['onhand'] - ideal_coin_debt
])
self.ideal_repay[coin] = ideal_repay_coin \
if ideal_repay_coin > approx_small_trade_amount * 2 else 0.0
####################
# big orders #
# long_sel #
if s in self.do_long_sel:
long_sel_amount = ((self.tradable_bnb if coin == 'BNB' else
self.balance[coin]['onhand']) -
self.ideal_shrt_sel[s]['amount'] -
self.ideal_repay[coin] +
self.ideal_borrow[coin])
long_sel_amount = max(
0.0, round_dn(long_sel_amount, self.amount_precisions[s]))
if long_sel_amount > min_big_trade_amount:
long_sel_price = max([
round_up(self.cm.max_ema[s], self.price_precisions[s]),
other_bid_incr,
(other_ask_decr
if long_sel_amount < lowest_other_ask['amount'] else
lowest_other_ask['price']),
round_up((self.my_trades_analyses[s]['true_long_vwap'] *
self.hyperparams['profit_pct_plus']),
self.price_precisions[s])
])
else:
long_sel_price = 0.0
self.ideal_long_sel[s] = {
'side': 'sell',
'amount': long_sel_amount,
'price': long_sel_price
}
else:
self.ideal_long_sel[s] = {
'side': 'sell',
'amount': 0.0,
'price': 0.0
}
############
# shrt_buy #
if s in self.do_shrt_buy:
shrt_buy_amount = (self.balance[coin]['borrowed'] +
self.ideal_borrow[coin] -
self.ideal_repay[coin] -
self.ideal_shrt_sel[s]['amount'])
shrt_buy_amount = max(0.0, round_dn(shrt_buy_amount))
shrt_buy_price = min([
round_dn(self.cm.min_ema[s], self.price_precisions[s]),
other_ask_decr,
(other_bid_incr
if shrt_buy_amount < highest_other_bid['amount'] else
highest_other_bid['price']),
round_dn((self.my_trades_analyses[s]['true_shrt_vwap'] *
self.hyperparams['profit_pct_minus']),
self.price_precisions[s]),
])
if shrt_buy_amount > min_big_trade_amount:
if self.balance[quot]['onhand'] + self.ideal_borrow[
quot] < ideal_quot_onhand:
# means we are max leveraged
shrt_buys_sorted_by_price_diff = sorted(
[
e for e in all_shrt_buys
if e['cost'] > small_trade_cost_default * 6
],
key=lambda x: self.cm.last_price[x['symbol']] / x[
'price'])
tmp_sum = quot_locked_in_long_buys
eligible_shrt_buys = []
for sb in shrt_buys_sorted_by_price_diff:
tmp_sum += sb['cost']
if tmp_sum >= self.balance[quot]['onhand']:
break
eligible_shrt_buys.append(sb)
if s not in set(
map(lambda x: x['symbol'], eligible_shrt_buys)):
shrt_buy_price = 0.0
else:
shrt_buy_price = 0.0
self.ideal_shrt_buy[s] = {
'side': 'buy',
'amount': shrt_buy_amount,
'price': shrt_buy_price
}
else:
self.ideal_shrt_buy[s] = {
'side': 'buy',
'amount': 0.0,
'price': 0.0
}
def backtest(df: pd.DataFrame, settings: dict):
symbols = list(df.symbol.unique())
quot = symbols[0].split('/')[1]
df_mid = df.iloc[int(len(df) * 0.5):int(len(df) * 0.55)]
precisions = {
s: find_precision(df_mid[df_mid.symbol == s].price.values)
for s in symbols
}
ppctminus = {
f'{c}/{quot}': 1 - settings['coins'][c]['profit_pct']
for c in settings['coins']
}
ppctplus = {
f'{c}/{quot}': 1 + settings['coins'][c]['profit_pct']
for c in settings['coins']
}
s2c = {s: s.split('/')[0] for s in symbols}
balance = {s2c[s]: 0.0 for s in s2c}
balance[quot] = settings['start_quot']
balance_ito_quot = {c: balance[c] for c in balance}
acc_equity_quot = settings['start_quot']
acc_debt_quot = 0.0
entries_list = []
exits_list = []
exit_prices_list = []
prev_long_entry_ts = {s: 0 for s in symbols}
prev_shrt_entry_ts = {s: 0 for s in symbols}
long_cost = {s: 0.0 for s in symbols}
long_amount = {s: 0.0 for s in symbols}
shrt_cost = {s: 0.0 for s in symbols}
shrt_amount = {s: 0.0 for s in symbols}
fee = 1 - settings['fee']
margin = settings['margin'] - 1
exponent = settings['entry_vol_modifier_exponent']
max_multiplier = settings['min_exit_cost_multiplier'] / 2
account_equity_pct_per_symbol_per_hour = {
c: settings['coins'][c]['account_equity_pct_per_hour']
for c in settings['coins']
}
millis_wait_until_next_long_entry = {s: 0 for s in symbols}
millis_wait_until_next_shrt_entry = {s: 0 for s in symbols}
coins_long = set(
[c for c in settings['coins'] if settings['coins'][c]['long']])
coins_shrt = set(
[c for c in settings['coins'] if settings['coins'][c]['shrt']])
balance_list = []
last_price = {s: 0.0 for s in s2c}
start_ts, end_ts = df.index[0], df.index[-1]
ts_range = end_ts - start_ts
k = 0
hour_to_millis = 60 * 60 * 1000
for row in df.itertuples():
s = row.symbol
last_price[s] = row.price
coin = s2c[s]
default_cost = max(
acc_equity_quot *
settings['coins'][coin]['account_equity_pct_per_trade'],
settings['min_quot_cost'])
credit_avbl_quot = max(0.0, acc_equity_quot * margin - acc_debt_quot)
if row.entry:
if row.is_buyer_maker:
if coin in coins_long and \
row.Index - prev_long_entry_ts[s] >= millis_wait_until_next_long_entry[s]:
cost = default_cost
if long_cost[s] > 0.0:
long_exit_price = \
long_cost[s] / long_amount[s] if long_amount[s] > 0.0 else row.price
cost *= max(
1.0,
min(max_multiplier,
(long_exit_price / row.price)**exponent))
quot_avbl = max(0.0, balance[quot])
borrow_amount = max(
0.0, min(cost - quot_avbl, credit_avbl_quot))
cost = min(quot_avbl + borrow_amount, cost)
if cost >= settings['min_quot_cost']:
credit_avbl_quot -= borrow_amount
amount = cost / row.entry_price
balance[quot] -= cost
balance[coin] += amount * fee
entries_list.append({
'symbol': s,
'timestamp': row.Index,
'side': 'buy',
'amount': amount,
'price': row.entry_price,
'cost': cost
})
long_cost[s] += cost
long_amount[s] += amount
prev_long_entry_ts[s] = row.Index
millis_wait_until_next_long_entry[s] = (default_cost * hour_to_millis) / \
(account_equity_pct_per_symbol_per_hour[coin] * acc_equity_quot)
else:
if coin in coins_shrt and \
row.Index - prev_shrt_entry_ts[s] >= millis_wait_until_next_shrt_entry[s]:
cost = default_cost
if shrt_cost[s] > 0.0:
shrt_exit_price = \
shrt_cost[s] / shrt_amount[s] if shrt_amount[s] > 0.0 else row.price
cost *= max(
1.0,
min(max_multiplier,
(row.price / shrt_exit_price)**exponent))
coin_avbl_quot = max(0.0, balance[coin]) * row.entry_price
borrow_amount_quot = max(
0.0, min(cost - coin_avbl_quot, credit_avbl_quot))
cost = min(coin_avbl_quot + borrow_amount_quot, cost)
if cost >= settings['min_quot_cost']:
credit_avbl_quot -= borrow_amount_quot
amount = cost / row.entry_price
balance[coin] -= amount
balance[quot] += cost * fee
entries_list.append({
'symbol': s,
'timestamp': row.Index,
'side': 'sell',
'amount': amount,
'price': row.entry_price,
'cost': cost
})
shrt_cost[s] += cost
shrt_amount[s] += amount
prev_shrt_entry_ts[s] = row.Index
millis_wait_until_next_shrt_entry[s] = (default_cost * hour_to_millis) / \
(account_equity_pct_per_symbol_per_hour[coin] * acc_equity_quot)
bag_ratio = (
(long_amount[s] - shrt_amount[s]) * row.price) / acc_equity_quot
if row.is_buyer_maker:
min_exit_cost = default_cost * settings['min_exit_cost_multiplier']
try:
shrt_vwap = shrt_cost[s] / shrt_amount[s]
except ZeroDivisionError:
shrt_vwap = 10e10
profit_pct = 1 - min(
0.1,
max(settings['coins'][coin]['profit_pct'],
bag_ratio * settings['profit_pct_multiplier']))
exit_price = min(row.exit_price,
round_dn(shrt_vwap * profit_pct, precisions[s]))
exit_cost = shrt_amount[s] * exit_price
exit_prices_list.append({
'timestamp': row.Index,
'symbol': s,
'side': 'buy',
'price': exit_price
})
if coin in coins_shrt and shrt_amount[
s] * exit_price >= min_exit_cost:
if row.price < exit_price and exit_cost >= min_exit_cost:
quot_avbl = max(0.0, balance[quot])
borrow_amount = max(
0.0, min(exit_cost - quot_avbl, credit_avbl_quot))
exit_cost = min(quot_avbl + borrow_amount, exit_cost)
if exit_cost >= min_exit_cost:
balance[quot] -= exit_cost
exit_amount = exit_cost / exit_price
balance[coin] += exit_amount * fee
exits_list.append({
'symbol': s,
'timestamp': row.Index,
'side': 'buy',
'amount': exit_amount,
'price': exit_price,
'cost': exit_cost
})
shrt_amount[s] -= exit_amount
shrt_cost[s] -= exit_cost
if shrt_amount[s] <= 0.0 or shrt_cost[s] <= 0.0:
shrt_amount[s], shrt_cost[s] = 0.0, 0.0
else:
min_exit_cost = default_cost * settings['min_exit_cost_multiplier']
try:
long_vwap = long_cost[s] / max(long_amount[s], 9e-9)
except ZeroDivisionError:
long_vwap = 0.0
profit_pct = 1 + min(
0.1,
max(settings['coins'][coin]['profit_pct'],
-bag_ratio * settings['profit_pct_multiplier']))
exit_price = max(row.exit_price,
round_up(long_vwap * profit_pct, precisions[s]))
exit_prices_list.append({
'timestamp': row.Index,
'symbol': s,
'side': 'sell',
'price': exit_price
})
if coin in coins_long and long_amount[
s] * row.exit_price >= min_exit_cost:
exit_cost = long_amount[s] * exit_price
if coin in coins_long and row.price > exit_price and exit_cost >= min_exit_cost:
coin_avbl_quot = max(0.0, balance[coin]) * exit_price
borrow_amount_quot = max(
0.0, min(exit_cost - coin_avbl_quot, credit_avbl_quot))
exit_cost = min(coin_avbl_quot + borrow_amount_quot,
exit_cost)
if exit_cost >= min_exit_cost:
exit_amount = exit_cost / exit_price
balance[coin] -= exit_amount
balance[quot] += exit_cost * fee
exits_list.append({
'symbol': s,
'timestamp': row.Index,
'side': 'sell',
'amount': exit_amount,
'price': exit_price,
'cost': exit_cost
})
long_amount[s] -= exit_amount
long_cost[s] -= exit_cost
if long_amount[s] <= 0.0 or long_cost[s] <= 0.0:
long_amount[s], long_cost[s] = 0.0, 0.0
acc_equity_quot -= (balance_ito_quot[coin] + balance_ito_quot[quot])
acc_debt_quot -= -(min(0.0, balance_ito_quot[coin]) +
min(0.0, balance_ito_quot[quot]))
balance_ito_quot[coin] = balance[coin] * row.price
balance_ito_quot[quot] = balance[quot]
acc_equity_quot += (balance_ito_quot[coin] + balance_ito_quot[quot])
acc_debt_quot += -(min(0.0, balance_ito_quot[coin]) +
min(0.0, balance_ito_quot[quot]))
onhand_ito_quot = sum([max(0.0, v) for v in balance_ito_quot.values()])
margin_level = onhand_ito_quot / acc_debt_quot if acc_debt_quot > 0.0 else 9e9
if margin_level <= settings['liquidation_margin_level']:
print('\nliquidation!')
return balance_list, entries_list, exits_list, exit_prices_list
k += 1
if k % 5000 == 0:
acc_equity_quot = sum(balance_ito_quot.values())
acc_debt_quot = -sum(
[min(0.0, v) for v in balance_ito_quot.values()])
n_millis = row.Index - start_ts
n_days = n_millis / 1000 / 60 / 60 / 24
avg_daily_gain = (acc_equity_quot /
settings['start_quot'])**(1 / n_days)
bag_ratios = {
f'bag_ratio_{s2c[s]}':
((long_amount[s] - shrt_amount[s]) * last_price[s]) /
acc_equity_quot
for s in s2c
}
balance_list.append({
**balance_ito_quot,
**{
'acc_equity_quot': acc_equity_quot,
'acc_debt_quot': acc_debt_quot,
'onhand_ito_quot': onhand_ito_quot,
'credit_avbl_quot': credit_avbl_quot,
'avg_daily_gain': avg_daily_gain,
'timestamp': row.Index
},
**bag_ratios
})
line = f'\r{(n_millis / ts_range) * 100:.2f}% '
line += f'n_days {n_days:.2f} '
line += f'acc equity quot: {acc_equity_quot:.6f} '
line += f"avg daily gain: {avg_daily_gain:6f} "
line += f'cost {default_cost:.8f} '
line += f'credit_avbl_quot {credit_avbl_quot:.6f} '
line += f'margin_level {margin_level:.4f} '
sys.stdout.write(line)
sys.stdout.flush()
return balance_list, entries_list, exits_list, exit_prices_list
def backtest(df: pd.DataFrame, settings: dict):
symbols = settings['symbols']
precisions = settings['precisions']
s2c = {s: s.split('/')[0] for s in symbols}
coins = sorted(set(s2c.values()))
quot = symbols[0].split('/')[1]
assert all([s.split('/')[1] == quot for s in symbols])
# max percentage of total account equity same side entry volume per symbol per hour
account_equity_pct_per_hour = settings[
'max_entry_acc_val_pct_per_hour'] / len(symbols)
# max percentage of total account equity same side entry cost per symbol per entry
account_equity_pct_per_entry = account_equity_pct_per_hour * settings[
'min_entry_delay_hours']
entry_delay_millis = settings['min_entry_delay_hours'] * HOUR_TO_MILLIS
print('account_equity_pct_per_entry', account_equity_pct_per_entry)
print('account_equity_pct_per_hour', account_equity_pct_per_hour)
print('min_entry_delay_hours', settings['min_entry_delay_hours'])
margin_multiplier = settings['max_leverage'] - 1
exponent = settings['entry_vol_modifier_exponent']
min_exit_cost_multiplier = settings['min_exit_cost_multiplier']
n_days_to_min_markup = settings['n_days_to_min_markup']
fee = 0.999
equity = {coin: 0.0 for coin in coins}
equity[quot] = 1.0
equity_ito_quot = equity.copy()
account_equity = equity[quot]
debt = 0.0
debt_neg = 0.0
onhand = equity[quot]
long_cost = {s: 0.0 for s in symbols}
long_amount = {s: 0.0 for s in symbols}
shrt_cost = {s: 0.0 for s in symbols}
shrt_amount = {s: 0.0 for s in symbols}
logs = []
trades = {s: [] for s in symbols}
prev_entry_ts = {
'long': {s: 0
for s in symbols},
'shrt': {s: 0
for s in symbols}
}
prev_exit_ts = {
'long': {s: df.index[0]
for s in symbols},
'shrt': {s: df.index[0]
for s in symbols}
}
last_price = {s: 0.0 for s in symbols}
for s in symbols:
for row in df.itertuples():
if row.symbol == s:
last_price[s] = row.avg
break
last_price[f'{quot}/{quot}'] = 1.0
start_ts, end_ts = df.index[0], df.index[-1]
ts_range = end_ts - start_ts
k = 0
kn = len(df) // 2000
liquidation = False
for row in df.itertuples():
s = row.symbol
coin = s2c[s]
entry_cost = account_equity * account_equity_pct_per_entry
credit = account_equity * margin_multiplier - debt_neg
try:
long_vwap = long_cost[s] / long_amount[s]
except ZeroDivisionError:
long_vwap = row.avg
try:
shrt_vwap = shrt_cost[s] / shrt_amount[s]
except ZeroDivisionError:
shrt_vwap = row.avg
##### long exit #####
long_bag_duration_days = (row.Index -
prev_exit_ts['long'][s]) / DAY_TO_MILLIS
long_exit_markup = max(
settings['min_markup_pct'], settings['max_markup_pct'] *
(1 - (long_bag_duration_days / n_days_to_min_markup)))
long_exit_price = max(
round_up(long_vwap * (1 + long_exit_markup), precisions[s]),
row.exit_ask)
if settings['long'] and row.high > long_exit_price:
coin_avbl = credit / row.avg + max(0.0, equity[coin])
long_exit_amount = min(coin_avbl, long_amount[s])
long_exit_cost = long_exit_amount * long_exit_price
if long_exit_cost > entry_cost * min_exit_cost_multiplier:
equity[coin] -= long_exit_amount
equity[quot] += long_exit_cost * fee
trades[s].append({
'timestamp': row.Index,
'side': 'sel',
'type': 'exit',
'price': long_exit_price,
'amount': long_exit_amount,
'cost': long_exit_cost,
'fee': long_exit_cost * (fee - 1) * -1
})
if long_exit_amount < long_amount[s]:
# partial exit
long_cost[s] -= long_exit_cost
long_amount[s] -= long_exit_amount
else:
prev_exit_ts['long'][s] = row.Index
long_cost[s] = 0.0
long_amount[s] = 0.0
##### shrt exit #####
shrt_bag_duration_days = (row.Index -
prev_exit_ts['shrt'][s]) / DAY_TO_MILLIS
shrt_exit_markup = max(
settings['min_markup_pct'], settings['max_markup_pct'] *
(1 - (shrt_bag_duration_days / n_days_to_min_markup)))
shrt_exit_price = min(
round_dn(shrt_vwap * (1 - shrt_exit_markup), precisions[s]),
row.exit_bid)
if settings['shrt'] and row.low < shrt_exit_price:
quot_avbl = credit + max(0.0, equity[quot])
shrt_exit_amount = min(quot_avbl / shrt_exit_price, shrt_amount[s])
shrt_exit_cost = shrt_exit_amount * shrt_exit_price
if shrt_exit_cost > entry_cost * min_exit_cost_multiplier:
equity[quot] -= shrt_exit_cost
equity[coin] += shrt_amount[s] * fee
trades[s].append({
'timestamp': row.Index,
'side': 'buy',
'type': 'exit',
'price': shrt_exit_price,
'amount': shrt_amount[s],
'cost': shrt_exit_cost,
'fee': shrt_exit_cost * (fee - 1) * -1
})
if shrt_exit_amount < shrt_amount[s]:
# partial exit
shrt_cost[s] -= shrt_exit_cost
shrt_amount[s] -= shrt_exit_amount
else:
prev_exit_ts['shrt'][s] = row.Index
shrt_cost[s] = 0.0
shrt_amount[s] = 0.0
##### long entry #####
if settings['long'] and row.Index - prev_entry_ts['long'][
s] >= entry_delay_millis:
if row.low < row.entry_bid:
long_entry_cost = entry_cost * max(
1.0,
min(min_exit_cost_multiplier / 2,
(long_vwap / row.entry_bid)**exponent))
if credit + max(0.0, equity[quot]) >= long_entry_cost:
long_entry_amount = long_entry_cost / row.entry_bid
equity[quot] -= long_entry_cost
equity[coin] += long_entry_amount * fee
long_cost[s] += long_entry_cost
long_amount[s] += long_entry_amount
prev_entry_ts['long'][s] = row.Index
trades[s].append({
'timestamp': row.Index,
'side': 'buy',
'type': 'entry',
'price': row.entry_bid,
'long_vwap': long_cost[s] / long_amount[s],
'amount': long_entry_amount,
'cost': long_entry_cost,
'fee': long_entry_cost * (fee - 1) * -1
})
##### shrt entry #####
if settings['shrt'] and row.Index - prev_entry_ts['shrt'][
s] >= entry_delay_millis:
if row.high > row.entry_ask:
shrt_entry_cost = entry_cost * max(
1.0,
min(min_exit_cost_multiplier / 2,
(row.entry_ask / shrt_vwap)**exponent))
if credit + max(
0.0, equity[coin] * row.entry_ask) >= shrt_entry_cost:
shrt_entry_amount = shrt_entry_cost / row.entry_ask
equity[coin] -= shrt_entry_amount
equity[quot] += shrt_entry_cost * fee
shrt_cost[s] += shrt_entry_cost
shrt_amount[s] += shrt_entry_amount
prev_entry_ts['shrt'][s] = row.Index
trades[s].append({
'timestamp': row.Index,
'side': 'sel',
'type': 'entry',
'price': row.entry_ask,
'shrt_vwap': shrt_cost[s] / shrt_amount[s],
'amount': shrt_entry_amount,
'cost': shrt_entry_cost,
'fee': shrt_entry_cost * (fee - 1) * -1
})
account_equity -= (equity_ito_quot[coin] + equity_ito_quot[quot])
debt -= (min(0.0, equity_ito_quot[coin]) +
min(0.0, equity_ito_quot[quot]))
onhand -= (max(0.0, equity_ito_quot[coin]) +
max(0.0, equity_ito_quot[quot]))
equity_ito_quot[s2c[s]] = equity[s2c[s]] * row.avg
equity_ito_quot[quot] = equity[quot]
account_equity += (equity_ito_quot[coin] + equity_ito_quot[quot])
debt += (min(0.0, equity_ito_quot[coin]) +
min(0.0, equity_ito_quot[quot]))
onhand += (max(0.0, equity_ito_quot[coin]) +
max(0.0, equity_ito_quot[quot]))
debt_neg = round(max(0.0, abs(debt)), 4)
margin_level = min(5.0, onhand / debt_neg if debt_neg else 5.0)
if margin_level < 1.05:
print('\nliquidation!')
print(debt)
print(equity_ito_quot)
print(equity)
print(row)
liquidation = True
k = kn - 1
k += 1
if k % kn == 0:
log_entry = {
**{
'timestamp': row.Index,
'debt': debt_neg,
'onhand': onhand,
'credit': credit,
'margin_level': margin_level,
'equity': account_equity
},
**equity_ito_quot
}
logs.append(log_entry)
n_millis = row.Index - start_ts
n_days = n_millis / (1000 * 60 * 60 * 24)
adg = account_equity**(1 / n_days)
ayg = adg**365
line = f'{n_millis / ts_range:.2f} margin_level, {margin_level:.2f}'
line += f' equity {account_equity:.4f} credit {credit:4f} ayg {ayg:.6f}'
sys.stdout.write('\r' + line + ' ' * 8)
if liquidation:
break
return logs, trades