def calc_orders(self):
default_orders = super().calc_orders()
orders = []
remaining_cost = self.balance[self.quot]["onhand"]
for order in sorted(default_orders,
key=lambda x: calc_diff(x["price"], self.price)):
if order["price"] > min(
self.max_price,
round_dn(self.price * self.price_multiplier_up,
self.price_step),
):
print(f'price {order["price"]} too high')
continue
if order["price"] < max(
self.min_price,
round_up(self.price * self.price_multiplier_dn,
self.price_step),
):
print(f'price {order["price"]} too low')
continue
if order["side"] == "buy":
cost = qty_to_cost(order["qty"], order["price"], self.inverse,
self.c_mult)
if cost > remaining_cost:
adjusted_qty = round_dn(remaining_cost / order["price"],
self.qty_step)
min_entry_qty = calc_min_entry_qty(
order["price"],
self.inverse,
self.qty_step,
self.min_qty,
self.min_cost,
)
if adjusted_qty >= min_entry_qty:
orders.append({**order, **{"qty": adjusted_qty}})
remaining_cost = 0.0
else:
orders.append(order)
remaining_cost -= cost
else:
orders.append(order)
return orders
def dump_plots(result: dict, fdf: pd.DataFrame, sdf: pd.DataFrame, df: pd.DataFrame):
init(autoreset=True)
plt.rcParams['figure.figsize'] = [29, 18]
pd.set_option('precision', 10)
table = PrettyTable(["Metric", "Value"])
table.align['Metric'] = 'l'
table.align['Value'] = 'l'
table.title = 'Summary'
table.add_row(['Exchange', result['exchange'] if 'exchange' in result else 'unknown'])
table.add_row(['Market type', result['market_type'] if 'market_type' in result else 'unknown'])
table.add_row(['Symbol', result['symbol'] if 'symbol' in result else 'unknown'])
table.add_row(['No. days', round_dynamic(result['result']['n_days'], 6)])
table.add_row(['Starting balance', round_dynamic(result['result']['starting_balance'], 6)])
profit_color = Fore.RED if result['result']['final_balance'] < result['result']['starting_balance'] else Fore.RESET
table.add_row(['Final balance', f"{profit_color}{round_dynamic(result['result']['final_balance'], 6)}{Fore.RESET}"])
table.add_row(['Final equity', f"{profit_color}{round_dynamic(result['result']['final_equity'], 6)}{Fore.RESET}"])
table.add_row(['Net PNL + fees', f"{profit_color}{round_dynamic(result['result']['net_pnl_plus_fees'], 6)}{Fore.RESET}"])
table.add_row(['Total gain percentage', f"{profit_color}{round_dynamic(result['result']['gain'] * 100, 4)}%{Fore.RESET}"])
table.add_row(['Average daily gain percentage', f"{profit_color}{round_dynamic((result['result']['average_daily_gain']) * 100, 3)}%{Fore.RESET}"])
table.add_row(['Adjusted daily gain', f"{profit_color}{round_dynamic(result['result']['adjusted_daily_gain'], 6)}{Fore.RESET}"])
bankruptcy_color = Fore.RED if result['result']['closest_bkr'] < 0.4 else Fore.YELLOW if result['result']['closest_bkr'] < 0.8 else Fore.RESET
table.add_row(['Closest bankruptcy percentage', f"{bankruptcy_color}{round_dynamic(result['result']['closest_bkr'] * 100, 4)}%{Fore.RESET}"])
table.add_row([' ', ' '])
table.add_row(['Profit sum', f"{profit_color}{round_dynamic(result['result']['profit_sum'], 6)}{Fore.RESET}"])
table.add_row(['Loss sum', f"{Fore.RED}{round_dynamic(result['result']['loss_sum'], 6)}{Fore.RESET}"])
table.add_row(['Fee sum', round_dynamic(result['result']['fee_sum'], 6)])
table.add_row(['Lowest equity/balance ratio', round_dynamic(result['result']['eqbal_ratio_min'], 6)])
table.add_row(['Biggest psize', round_dynamic(result['result']['biggest_psize'], 6)])
table.add_row(['Price action distance mean long', round_dynamic(result['result']['pa_closeness_mean_long'], 6)])
table.add_row(['Price action distance median long', round_dynamic(result['result']['pa_closeness_median_long'], 6)])
table.add_row(['Price action distance max long', round_dynamic(result['result']['pa_closeness_max_long'], 6)])
table.add_row(['Price action distance mean short', round_dynamic(result['result']['pa_closeness_mean_shrt'], 6)])
table.add_row(['Price action distance median short', round_dynamic(result['result']['pa_closeness_median_shrt'], 6)])
table.add_row(['Price action distance max short', round_dynamic(result['result']['pa_closeness_max_shrt'], 6)])
table.add_row(['Average n fills per day', round_dynamic(result['result']['avg_fills_per_day'], 6)])
table.add_row([' ', ' '])
table.add_row(['No. fills', round_dynamic(result['result']['n_fills'], 6)])
table.add_row(['No. entries', round_dynamic(result['result']['n_entries'], 6)])
table.add_row(['No. closes', round_dynamic(result['result']['n_closes'], 6)])
table.add_row(['No. initial entries', round_dynamic(result['result']['n_ientries'], 6)])
table.add_row(['No. reentries', round_dynamic(result['result']['n_rentries'], 6)])
table.add_row([' ', ' '])
table.add_row(['Mean hours between fills', round_dynamic(result['result']['hrs_stuck_avg_long'], 6)])
table.add_row(['Max hours no fills (same side)', round_dynamic(result['result']['hrs_stuck_max_long'], 6)])
table.add_row(['Max hours no fills', round_dynamic(result['result']['hrs_stuck_max_long'], 6)])
longs = fdf[fdf.type.str.contains('long')]
shrts = fdf[fdf.type.str.contains('shrt')]
if result['long']['enabled']:
table.add_row([' ', ' '])
table.add_row(['Long', result['long']['enabled']])
table.add_row(["No. inital entries", len(longs[longs.type.str.contains('ientry')])])
table.add_row(["No. reentries", len(longs[longs.type.str.contains('rentry')])])
table.add_row(["No. normal closes", len(longs[longs.type.str.contains('nclose')])])
table.add_row(['Mean hours stuck (long)', round_dynamic(result['result']['hrs_stuck_avg_long'], 6)])
table.add_row(['Max hours stuck (long)', round_dynamic(result['result']['hrs_stuck_max_long'], 6)])
profit_color = Fore.RED if longs.pnl.sum() < 0 else Fore.RESET
table.add_row(["PNL sum", f"{profit_color}{longs.pnl.sum()}{Fore.RESET}"])
if result['shrt']['enabled']:
table.add_row([' ', ' '])
table.add_row(['Short', result['shrt']['enabled']])
table.add_row(["No. inital entries", len(shrts[shrts.type.str.contains('ientry')])])
table.add_row(["No. reentries", len(shrts[shrts.type.str.contains('rentry')])])
table.add_row(["No. normal closes", len(shrts[shrts.type.str.contains('nclose')])])
table.add_row(['Mean hours stuck (shrt)', round_dynamic(result['result']['hrs_stuck_avg_shrt'], 6)])
table.add_row(['Max hours stuck (shrt)', round_dynamic(result['result']['hrs_stuck_max_shrt'], 6)])
profit_color = Fore.RED if shrts.pnl.sum() < 0 else Fore.RESET
table.add_row(["PNL sum", f"{profit_color}{shrts.pnl.sum()}{Fore.RESET}"])
dump_live_config(result, result['plots_dirpath'] + 'live_config.json')
json.dump(denumpyize(result), open(result['plots_dirpath'] + 'result.json', 'w'), indent=4)
print('writing backtest_result.txt...\n')
with open(f"{result['plots_dirpath']}backtest_result.txt", 'w') as f:
output = table.get_string(border=True, padding_width=1)
print(output)
f.write(re.sub('\033\\[([0-9]+)(;[0-9]+)*m', '', output))
print('\nplotting balance and equity...')
plt.clf()
sdf.balance.plot()
sdf.equity.plot(title="Balance and equity", xlabel="Time", ylabel="Balance")
plt.savefig(f"{result['plots_dirpath']}balance_and_equity_sampled.png")
plt.clf()
longs.pnl.cumsum().plot(title="PNL cumulated sum - Long", xlabel="Time", ylabel="PNL")
plt.savefig(f"{result['plots_dirpath']}pnl_cumsum_long.png")
plt.clf()
shrts.pnl.cumsum().plot(title="PNL cumulated sum - Short", xlabel="Time", ylabel="PNL")
plt.savefig(f"{result['plots_dirpath']}pnl_cumsum_shrt.png")
adg = (sdf.equity / sdf.equity.iloc[0]) ** (1 / ((sdf.timestamp - sdf.timestamp.iloc[0]) / (1000 * 60 * 60 * 24)))
plt.clf()
adg.plot(title="Average daily gain", xlabel="Time", ylabel="Average daily gain")
plt.savefig(f"{result['plots_dirpath']}adg.png")
print('plotting backtest whole and in chunks...')
n_parts = max(3, int(round_up(result['n_days'] / 14, 1.0)))
for z in range(n_parts):
start_ = z / n_parts
end_ = (z + 1) / n_parts
print(f'{z} of {n_parts} {start_ * 100:.2f}% to {end_ * 100:.2f}%')
fig = plot_fills(df, fdf.iloc[int(len(fdf) * start_):int(len(fdf) * end_)], bkr_thr=0.1, title=f'Fills {z+1} of {n_parts}')
if fig is not None:
fig.savefig(f"{result['plots_dirpath']}backtest_{z + 1}of{n_parts}.png")
else:
print('no fills...')
fig = plot_fills(df, fdf, bkr_thr=0.1, plot_whole_df=True, title='Overview Fills')
fig.savefig(f"{result['plots_dirpath']}whole_backtest.png")
print('plotting pos sizes...')
plt.clf()
longs.psize.plot()
shrts.psize.plot(title="Position size in terms of contracts", xlabel="Time", ylabel="Position size")
plt.savefig(f"{result['plots_dirpath']}psizes_plot.png")
def calc_recursive_entry_short(
balance,
psize,
pprice,
lowest_ask,
ema_band_upper,
inverse,
qty_step,
price_step,
min_qty,
min_cost,
c_mult,
initial_qty_pct,
initial_eprice_ema_dist,
ddown_factor,
rentry_pprice_dist,
rentry_pprice_dist_wallet_exposure_weighting,
wallet_exposure_limit,
auto_unstuck_ema_dist,
auto_unstuck_wallet_exposure_threshold,
):
abs_psize = abs(psize)
ientry_price = max(
lowest_ask,
round_up(ema_band_upper * (1 + initial_eprice_ema_dist), price_step))
min_entry_qty = calc_min_entry_qty(ientry_price, inverse, qty_step,
min_qty, min_cost)
ientry_qty = max(
min_entry_qty,
round_(
cost_to_qty(balance, ientry_price, inverse, c_mult) *
wallet_exposure_limit * initial_qty_pct,
qty_step,
),
)
if abs_psize == 0.0:
# normal ientry
return -ientry_qty, ientry_price, "short_ientry_normal"
elif abs_psize < ientry_qty * 0.8:
# partial ientry
entry_qty = max(min_entry_qty, round_(ientry_qty - abs_psize,
qty_step))
return -entry_qty, ientry_price, "short_ientry_partial"
else:
wallet_exposure = qty_to_cost(abs_psize, pprice, inverse,
c_mult) / balance
if wallet_exposure >= wallet_exposure_limit * 1.001:
# no entry if wallet_exposure within 0.1% of limit
return 0.0, 0.0, ""
threshold = wallet_exposure_limit * (
1 - auto_unstuck_wallet_exposure_threshold)
if auto_unstuck_wallet_exposure_threshold != 0.0 and wallet_exposure > threshold * 0.99:
# auto unstuck mode
entry_price = round_up(
max([
lowest_ask, pprice,
ema_band_upper * (1 + auto_unstuck_ema_dist)
]), price_step)
entry_qty = find_entry_qty_bringing_wallet_exposure_to_target(
balance,
abs_psize,
pprice,
wallet_exposure_limit,
entry_price,
inverse,
qty_step,
c_mult,
)
min_entry_qty = calc_min_entry_qty(entry_price, inverse, qty_step,
min_qty, min_cost)
return (-max(entry_qty, min_entry_qty), entry_price,
"short_unstuck_entry")
else:
# normal reentry
ratio = wallet_exposure / wallet_exposure_limit
entry_price = round_up(
pprice *
(1 + rentry_pprice_dist *
(1 + ratio * rentry_pprice_dist_wallet_exposure_weighting)),
price_step,
)
entry_price = max(entry_price, lowest_ask)
min_entry_qty = calc_min_entry_qty(entry_price, inverse, qty_step,
min_qty, min_cost)
entry_qty = max(min_entry_qty,
round_(abs_psize * ddown_factor, qty_step))
wallet_exposure_if_filled = calc_wallet_exposure_if_filled(
balance, abs_psize, pprice, entry_qty, entry_price, inverse,
c_mult, qty_step)
if wallet_exposure_if_filled > wallet_exposure_limit * 1.01:
# crop qty
entry_qty = find_entry_qty_bringing_wallet_exposure_to_target(
balance,
abs_psize,
pprice,
wallet_exposure_limit,
entry_price,
inverse,
qty_step,
c_mult,
)
entry_qty = max(entry_qty, min_entry_qty)
return -entry_qty, entry_price, "short_rentry"
def dump_plots(
result: dict,
longs: pd.DataFrame,
shorts: pd.DataFrame,
sdf: pd.DataFrame,
df: pd.DataFrame,
n_parts: int = None,
):
init(autoreset=True)
plt.rcParams["figure.figsize"] = [29, 18]
try:
pd.set_option("display.precision", 10)
except Exception as e:
print("error setting pandas precision", e)
table = PrettyTable(["Metric", "Value"])
table.align["Metric"] = "l"
table.align["Value"] = "l"
table.title = "Summary"
table.add_row([
"Exchange", result["exchange"] if "exchange" in result else "unknown"
])
table.add_row([
"Market type",
result["market_type"] if "market_type" in result else "unknown"
])
table.add_row(
["Symbol", result["symbol"] if "symbol" in result else "unknown"])
table.add_row(["No. days", round_dynamic(result["result"]["n_days"], 6)])
table.add_row([
"Starting balance",
round_dynamic(result["result"]["starting_balance"], 6)
])
for side in ["long", "short"]:
if result[side]["enabled"]:
table.add_row([" ", " "])
table.add_row([side.capitalize(), result[side]["enabled"]])
adg_per_exp = result["result"][f"adg_{side}"] / result[side][
"wallet_exposure_limit"]
table.add_row([
"ADG per exposure", f"{round_dynamic(adg_per_exp * 100, 3)}%"
])
profit_color = (
Fore.RED if result["result"][f"final_balance_{side}"] <
result["result"]["starting_balance"] else Fore.RESET)
table.add_row([
"Final balance",
f"{profit_color}{round_dynamic(result['result'][f'final_balance_{side}'], 6)}{Fore.RESET}",
])
table.add_row([
"Final equity",
f"{profit_color}{round_dynamic(result['result'][f'final_equity_{side}'], 6)}{Fore.RESET}",
])
table.add_row([
"Net PNL + fees",
f"{profit_color}{round_dynamic(result['result'][f'net_pnl_plus_fees_{side}'], 6)}{Fore.RESET}",
])
table.add_row([
"Total gain",
f"{profit_color}{round_dynamic(result['result'][f'gain_{side}'] * 100, 4)}%{Fore.RESET}",
])
table.add_row([
"Average daily gain",
f"{profit_color}{round_dynamic((result['result'][f'adg_{side}']) * 100, 3)}%{Fore.RESET}",
])
gain_per_exp = result["result"][f"gain_{side}"] / result[side][
"wallet_exposure_limit"]
table.add_row([
"Gain per exposure", f"{round_dynamic(gain_per_exp * 100, 3)}%"
])
table.add_row([
"DG mean std ratio",
f"{round_dynamic(result['result'][f'adg_DGstd_ratio_{side}'], 4)}",
])
table.add_row([
f"Price action distance mean",
round_dynamic(result["result"][f"pa_distance_mean_{side}"], 6),
])
table.add_row([
f"Price action distance std",
round_dynamic(result["result"][f"pa_distance_std_{side}"], 6),
])
table.add_row([
f"Price action distance max",
round_dynamic(result["result"][f"pa_distance_max_{side}"], 6),
])
table.add_row([
"Closest bankruptcy",
f'{round_dynamic(result["result"][f"closest_bkr_{side}"] * 100, 4)}%',
])
table.add_row([
"Lowest equity/balance ratio",
f'{round_dynamic(result["result"][f"eqbal_ratio_min_{side}"], 4)}',
])
table.add_row(["No. fills", result["result"][f"n_fills_{side}"]])
table.add_row(
["No. entries", result["result"][f"n_entries_{side}"]])
table.add_row(["No. closes", result["result"][f"n_closes_{side}"]])
table.add_row([
"No. initial entries", result["result"][f"n_ientries_{side}"]
])
table.add_row(
["No. reentries", result["result"][f"n_rentries_{side}"]])
table.add_row([
"No. unstuck entries",
result["result"][f"n_unstuck_entries_{side}"]
])
table.add_row([
"No. unstuck closes",
result["result"][f"n_unstuck_closes_{side}"]
])
table.add_row([
"No. normal closes",
result["result"][f"n_normal_closes_{side}"]
])
table.add_row([
"Average n fills per day",
round_dynamic(result["result"][f"avg_fills_per_day_{side}"],
3),
])
table.add_row([
"Mean hours stuck",
round_dynamic(result["result"][f"hrs_stuck_avg_{side}"], 6),
])
table.add_row([
"Max hours stuck",
round_dynamic(result["result"][f"hrs_stuck_max_{side}"], 6),
])
profit_color = Fore.RED if result["result"][
f"pnl_sum_{side}"] < 0 else Fore.RESET
table.add_row([
"PNL sum",
f"{profit_color}{round_dynamic(result['result'][f'pnl_sum_{side}'], 4)}{Fore.RESET}",
])
table.add_row([
"Profit sum",
round_dynamic(result["result"][f"profit_sum_{side}"], 4)
])
table.add_row([
"Loss sum",
round_dynamic(result["result"][f"loss_sum_{side}"], 4)
])
table.add_row([
"Fee sum",
round_dynamic(result["result"][f"fee_sum_{side}"], 4)
])
table.add_row([
"Biggest pos size", result["result"][f"biggest_psize_{side}"]
])
table.add_row([
"Biggest pos cost",
round_dynamic(result["result"][f"biggest_psize_quote_{side}"],
4),
])
table.add_row([
"Volume quote",
round_dynamic(result["result"][f"volume_quote_{side}"], 6)
])
dump_live_config(result, result["plots_dirpath"] + "live_config.json")
json.dump(denumpyize(result),
open(result["plots_dirpath"] + "result.json", "w"),
indent=4)
print("writing backtest_result.txt...\n")
with open(f"{result['plots_dirpath']}backtest_result.txt", "w") as f:
output = table.get_string(border=True, padding_width=1)
print(output)
f.write(re.sub("\033\\[([0-9]+)(;[0-9]+)*m", "", output))
n_parts = n_parts if n_parts is not None else max(
3, int(round_up(result["n_days"] / 14, 1.0)))
for side, fdf in [("long", longs), ("short", shorts)]:
if result[side]["enabled"]:
plt.clf()
fig = plot_fills(df,
fdf,
plot_whole_df=True,
title=f"Overview Fills {side.capitalize()}")
fig.savefig(f"{result['plots_dirpath']}whole_backtest_{side}.png")
print(f"\nplotting balance and equity {side}...")
plt.clf()
sdf[f"balance_{side}"].plot()
sdf[f"equity_{side}"].plot(
title=f"Balance and equity {side.capitalize()}",
xlabel="Time",
ylabel="Balance")
plt.savefig(
f"{result['plots_dirpath']}balance_and_equity_sampled_{side}.png"
)
for z in range(n_parts):
start_ = z / n_parts
end_ = (z + 1) / n_parts
print(
f"{side} {z} of {n_parts} {start_ * 100:.2f}% to {end_ * 100:.2f}%"
)
fig = plot_fills(
df,
fdf.iloc[int(len(fdf) * start_):int(len(fdf) * end_)],
title=f"Fills {side} {z+1} of {n_parts}",
)
if fig is not None:
fig.savefig(
f"{result['plots_dirpath']}backtest_{side}{z + 1}of{n_parts}.png"
)
else:
print(f"no {side} fills...")
print(f"plotting {side} initial entry band")
spans_multiplier = 60 / ((df.index[1] - df.index[0]) / 1000)
spans = [
result[side]["ema_span_0"] * spans_multiplier,
((result[side]["ema_span_0"] * result[side]["ema_span_1"])**
0.5) * spans_multiplier,
result[side]["ema_span_1"] * spans_multiplier,
]
emas = pd.DataFrame({
str(span): df.iloc[::100].price.ewm(span=max(1.0, span / 100),
adjust=False).mean()
for span in spans
})
ema_band_lower = emas.min(axis=1)
ema_band_upper = emas.max(axis=1)
if side == "long":
ientry_band = ema_band_lower * (
1 - result[side]["initial_eprice_ema_dist"])
else:
ientry_band = ema_band_upper * (
1 + result[side]["initial_eprice_ema_dist"])
plt.clf()
df.price.iloc[::100].plot(
style="y-", title=f"{side.capitalize()} Initial Entry Band")
ientry_band.plot(style=f"{('b' if side == 'long' else 'r')}-.")
plt.savefig(
f"{result['plots_dirpath']}initial_entry_band_{side}.png")
if result[side]["auto_unstuck_wallet_exposure_threshold"] != 0.0:
print(f"plotting {side} unstucking bands...")
unstucking_band_lower = ema_band_lower * (
1 - result[side]["auto_unstuck_ema_dist"])
unstucking_band_upper = ema_band_upper * (
1 + result[side]["auto_unstuck_ema_dist"])
plt.clf()
df.price.iloc[::100].plot(
style="y-",
title=f"{side.capitalize()} Auto Unstucking Bands")
unstucking_band_lower.plot(style="b-.")
unstucking_band_upper.plot(style="r-.")
plt.savefig(
f"{result['plots_dirpath']}auto_unstuck_bands_{side}.png")
print("plotting pos sizes...")
plt.clf()
sdf[["psize_long", "psize_short"]].plot(
title="Position size in terms of contracts",
xlabel="Time",
ylabel="Position size",
)
plt.savefig(f"{result['plots_dirpath']}psizes_plot.png")