async def main():
exchange = sys.argv[1]
user = sys.argv[2]
settings_filepath = os.path.join('backtesting_settings', exchange, '')
backtesting_settings = \
json.load(open(os.path.join(settings_filepath, 'backtesting_settings.json')))
symbol = backtesting_settings['symbol']
n_days = backtesting_settings['n_days']
ranges = json.load(open(os.path.join(settings_filepath, 'ranges.json')))
print(settings_filepath)
results_filepath = make_get_filepath(
os.path.join(
'backtesting_results', exchange,
ts_to_date(time())[:19].replace(':', '_') +
f'_{int(round(n_days))}', ''))
trade_cache_filepath = make_get_filepath(
os.path.join(settings_filepath, 'trade_cache', ''))
trades_filename = f'{symbol}_raw_trades_{exchange}_{n_days}_days_{ts_to_date(time())[:10]}.npy'
trades_filepath = f"{trade_cache_filepath}{trades_filename}"
if os.path.exists(trades_filepath):
print('loading cached trade list', trades_filepath)
trades_list = np.load(trades_filepath, allow_pickle=True)
else:
agg_trades = await load_trades(exchange, user, symbol, n_days)
print('preparing trades...')
trades_list = prep_trades_list(agg_trades)
np.save(trades_filepath, trades_list)
jackrabbit(trades_list, backtesting_settings, ranges, results_filepath)
async def main():
exchange = sys.argv[1]
user = sys.argv[2]
settings_filepath = os.path.join('backtesting_settings', exchange, '')
backtesting_settings = \
json.load(open(os.path.join(settings_filepath, 'backtesting_settings.json')))
try:
session_name = sys.argv[3]
print('\n\nusing given session name', session_name, '\n\n')
except IndexError:
session_name = backtesting_settings['session_name']
print('\n\nusing session name from backtesting_settings.json',
session_name, '\n\n')
symbol = backtesting_settings['symbol']
n_days = backtesting_settings['n_days']
ranges = json.load(open(os.path.join(settings_filepath, 'ranges.json')))
print(settings_filepath)
results_filepath = make_get_filepath(
os.path.join('backtesting_results', exchange, symbol, session_name,
''))
print(results_filepath)
trades_list_filepath = os.path.join(
results_filepath, f"{n_days}_days_trades_list_cache.npy")
if os.path.exists(trades_list_filepath):
print('loading cached trade list', trades_list_filepath)
trades_list = np.load(trades_list_filepath, allow_pickle=True)
else:
agg_trades = await load_trades(exchange, user, symbol, n_days)
print('preparing trades...')
trades_list = prep_trades_list(agg_trades)
np.save(trades_list_filepath, trades_list)
jackrabbit(trades_list, backtesting_settings, ranges, results_filepath)
def plot_wrap(bc, ticks, candidate):
n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1)
print('backtesting...')
result, fdf = backtest_wrap(ticks, {
**bc,
**{
'break_on': {}
},
**candidate
},
do_print=True)
if fdf is None or len(fdf) == 0:
print('no trades')
return
backtest_config = {**bc, **candidate, **result}
backtest_config['session_dirpath'] = make_get_filepath(
os.path.join(
'plots', bc['exchange'], bc['symbol'],
f"{n_days}_days_{ts_to_date(time())[:19].replace(':', '')}", ''))
fdf.to_csv(backtest_config['session_dirpath'] +
f"backtest_trades_{result['key']}.csv")
df = pd.DataFrame({
'price': ticks[:, 0],
'buyer_maker': ticks[:, 1],
'timestamp': ticks[:, 2]
})
dump_plots(backtest_config, fdf, df)
def backtest_tune(ticks: np.ndarray, backtest_config: dict, current_best: Union[dict, list] = None):
config = create_config(backtest_config)
n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1)
session_dirpath = make_get_filepath(os.path.join('reports', backtest_config['exchange'], backtest_config['symbol'],
f"{n_days}_days_{ts_to_date(time())[:19].replace(':', '')}", ''))
iters = 10
if 'iters' in backtest_config:
iters = backtest_config['iters']
else:
print('Parameter iters should be defined in the configuration. Defaulting to 10.')
num_cpus = 2
if 'num_cpus' in backtest_config:
num_cpus = backtest_config['num_cpus']
else:
print('Parameter num_cpus should be defined in the configuration. Defaulting to 2.')
n_particles = 10
if 'n_particles' in backtest_config:
n_particles = backtest_config['n_particles']
phi1 = 1.4962
phi2 = 1.4962
omega = 0.7298
if 'options' in backtest_config:
phi1 = backtest_config['options']['c1']
phi2 = backtest_config['options']['c2']
omega = backtest_config['options']['w']
current_best_params = []
if current_best:
if type(current_best) == list:
for c in current_best:
c = clean_start_config(c, config, backtest_config['ranges'])
current_best_params.append(c)
else:
current_best = clean_start_config(current_best, config, backtest_config['ranges'])
current_best_params.append(current_best)
ray.init(num_cpus=num_cpus, logging_level=logging.FATAL, log_to_driver=False)
pso = ng.optimizers.ConfiguredPSO(transform='identity', popsize=n_particles, omega=omega, phip=phi1, phig=phi2)
algo = NevergradSearch(optimizer=pso, points_to_evaluate=current_best_params)
algo = ConcurrencyLimiter(algo, max_concurrent=num_cpus)
scheduler = AsyncHyperBandScheduler()
analysis = tune.run(tune.with_parameters(wrap_backtest, ticks=ticks), metric='objective', mode='max', name='search',
search_alg=algo, scheduler=scheduler, num_samples=iters, config=config, verbose=1,
reuse_actors=True, local_dir=session_dirpath,
progress_reporter=LogReporter(metric_columns=['daily_gain',
'closest_liquidation',
'max_hours_between_fills',
'objective'],
parameter_columns=[k for k in backtest_config['ranges'] if type(
config[k]) == ray.tune.sample.Float or type(
config[k]) == ray.tune.sample.Integer]))
ray.shutdown()
return analysis
def plot_wrap(bc, ticks, live_config):
n_days = round_((ticks[-1][2] - ticks[0][2]) / (1000 * 60 * 60 * 24), 0.1)
print('n_days', round_(n_days, 0.1))
config = {**bc, **live_config}
print('backtesting...')
fills, stats, did_finish = backtest(config, ticks, do_print=True)
if not fills:
print('no fills')
return
fdf, result = analyze_fills(fills, config, ticks[-1][2])
config['result'] = result
config['plots_dirpath'] = make_get_filepath(os.path.join(
config['plots_dirpath'], f"{ts_to_date(time())[:19].replace(':', '')}", '')
)
fdf.to_csv(config['plots_dirpath'] + "fills.csv")
df = pd.DataFrame({'price': ticks[:, 0], 'buyer_maker': ticks[:, 1], 'timestamp': ticks[:, 2]})
dump_plots(config, fdf, df)
async def load_trades(exchange: str, user: str, symbol: str,
n_days: float) -> pd.DataFrame:
def skip_ids(id_, ids_):
if id_ in ids_:
print('skipping from', id_)
while id_ in ids_:
id_ -= 1
print(' to', id_)
return id_
def load_cache():
cache_filenames = [
f for f in os.listdir(cache_filepath) if '.csv' in f
]
if cache_filenames:
print('loading cached trades')
cache_df = pd.concat(
[pd.read_csv(cache_filepath + f) for f in cache_filenames],
axis=0)
cache_df = cache_df.set_index('trade_id')
return cache_df
return None
if exchange == 'binance':
fetch_trades_func = binance_fetch_trades
elif exchange == 'bybit':
fetch_trades_func = bybit_fetch_trades
else:
print(exchange, 'not found')
return
cc = init_ccxt(exchange, user)
filepath = make_get_filepath(
os.path.join('historical_data', exchange, 'agg_trades_futures', symbol,
''))
cache_filepath = make_get_filepath(
filepath.replace(symbol, symbol + '_cache'))
age_limit = time() - 60 * 60 * 24 * n_days
age_limit_millis = age_limit * 1000
print('age_limit', ts_to_date(age_limit))
cache_df = load_cache()
trades_df, chunk_lengths = get_downloaded_trades(filepath,
age_limit_millis)
ids = set()
if trades_df is not None:
ids.update(trades_df.index)
if cache_df is not None:
ids.update(cache_df.index)
gaps = []
if trades_df is not None and len(trades_df) > 0:
#
sids = sorted(ids)
for i in range(1, len(sids)):
if sids[i - 1] + 1 != sids[i]:
gaps.append((sids[i - 1], sids[i]))
if gaps:
print('gaps', gaps)
#
prev_fetch_ts = time()
new_trades = await fetch_trades_func(cc, symbol)
k = 0
while True:
k += 1
if (break_ :=
new_trades[0]['timestamp'] <= age_limit_millis) or k % 20 == 0:
print('caching trades...')
new_tdf = pd.DataFrame(new_trades).set_index('trade_id')
cache_filename = f'{cache_filepath}{new_tdf.index[0]}_{new_tdf.index[-1]}.csv'
new_tdf.to_csv(cache_filename)
new_trades = [new_trades[0]]
if break_:
break
from_id = skip_ids(new_trades[0]['trade_id'] - 1, ids) - 999
# wait at least 0.75 sec between each fetch
sleep_for = max(0.0, 0.75 - (time() - prev_fetch_ts))
await asyncio.sleep(sleep_for)
prev_fetch_ts = time()
new_trades = await fetch_trades_func(cc, symbol,
from_id=from_id) + new_trades
ids.update([e['trade_id'] for e in new_trades])
def jackrabbit(trades_list: [dict], backtesting_settings: dict, ranges: dict,
base_filepath: str):
ks = backtesting_settings['n_jackrabbit_iterations']
k = backtesting_settings['starting_k']
ms = np.array([1 / (i / 2 + 16) for i in range(ks)])
ms = ((ms - ms.min()) / (ms.max() - ms.min()))
best_filepath = base_filepath + 'best.json'
if backtesting_settings['starting_candidate_preference'][0] == 'best' and \
os.path.exists(best_filepath):
best = json.load(open(best_filepath))
candidate = get_new_candidate(ranges, best, ms[k])
print('\ncurrent best')
print(json.dumps(best, indent=4, sort_keys=True))
else:
best = {k_: backtesting_settings[k_] for k_ in ranges}
best['gain'] = -9e9
if 'given' in backtesting_settings[
'starting_candidate_preference'][:2]:
candidate = best.copy()
print('\nusing starting candidate from backtesting_settings')
else:
candidate = get_new_candidate(ranges, best, m=1.0)
print('\nusing random starting candidate')
print(json.dumps(candidate, indent=4, sort_keys=True))
results = {}
n_days = backtesting_settings['n_days']
trades_filepath = make_get_filepath(
os.path.join(base_filepath, 'backtest_trades', ''))
results_filename = base_filepath + 'results.txt'
json.dump(backtesting_settings,
open(base_filepath + 'backtesting_settings.json', 'w'),
indent=4,
sort_keys=True)
json.dump(ranges,
open(base_filepath + 'ranges.json', 'w'),
indent=4,
sort_keys=True)
while k < ks:
mutation_coefficient = ms[k]
if candidate['min_markup'] >= candidate['max_markup']:
candidate['min_markup'] = candidate['max_markup']
settings = {**backtesting_settings, **candidate}
key = np.format_float_positional(hash(
json.dumps({k_: candidate[k_]
for k_ in ranges})),
trim='-')[1:20]
if key in results:
if os.path.exists(best_filepath):
best = json.load(open(best_filepath))
candidate = get_new_candidate(ranges, best, mutation_coefficient)
continue
print(f'\nk={k}, m={mutation_coefficient:.4f} candidate:\n', candidate)
start_time = time()
trades = backtest(trades_list, settings)
print('\ntime elapsed', round(time() - start_time, 1), 'seconds')
k += 1
if not trades:
print('\nno trades')
if os.path.exists(best_filepath):
best = json.load(open(best_filepath))
candidate = get_new_candidate(ranges, best, mutation_coefficient)
continue
tdf = pd.DataFrame(trades).set_index('trade_id')
closest_liq = ((tdf.price - tdf.liq_price).abs() / tdf.price).min()
biggest_pos_size = tdf.pos_size.abs().max()
n_closes = len(tdf[tdf.type == 'close'])
pnl_sum = tdf.pnl.sum()
loss_sum = tdf[tdf.type == 'stop_loss'].pnl.sum()
abs_pos_sizes = tdf.pos_size.abs()
gain = (pnl_sum +
settings['starting_balance']) / settings['starting_balance']
candidate['gain'] = gain
average_daily_gain = gain**(1 / n_days)
n_trades = len(tdf)
result = {
'n_closes': n_closes,
'pnl_sum': pnl_sum,
'loss_sum': loss_sum,
'average_daily_gain': average_daily_gain,
'gain': gain,
'n_trades': n_trades,
'closest_liq': closest_liq,
'biggest_pos_size': biggest_pos_size,
'n_days': n_days,
'key': key
}
tdf.to_csv(f'{trades_filepath}{key}.csv')
print('\n\n', result)
results[key] = {**result, **candidate}
if os.path.exists(best_filepath):
best = json.load(open(best_filepath))
if candidate['gain'] > best['gain']:
best = candidate
print('\n\n\n###############\nnew best', best,
'\naverage daily gain:', round(average_daily_gain,
5), '\n\n')
print(settings, '\n')
print(results[key], '\n\n')
default_live_settings = load_settings(settings['exchange'],
do_print=False)
live_settings = {
k: settings[k] if k in settings else default_live_settings[k]
for k in default_live_settings
}
live_settings['indicator_settings'] = {
'tick_ema': {
'span': best['ema_span']
},
'do_long': backtesting_settings['do_long'],
'do_shrt': backtesting_settings['do_shrt']
}
json.dump(live_settings,
open(base_filepath + 'best_result_live_settings.json',
'w'),
indent=4,
sort_keys=True)
print('\n\n', json.dumps(live_settings, indent=4, sort_keys=True),
'\n\n')
json.dump({
**{
'gain': result['gain']
},
**best
},
open(best_filepath, 'w'),
indent=4,
sort_keys=True)
candidate = get_new_candidate(ranges, best, m=mutation_coefficient)
with open(results_filename, 'a') as f:
f.write(json.dumps(results[key]) + '\n')
async def main():
exchange = sys.argv[1]
user = sys.argv[2]
settings_filepath = os.path.join('backtesting_settings', exchange, '')
backtesting_settings = \
json.load(open(os.path.join(settings_filepath, 'backtesting_settings.json')))
try:
session_name = sys.argv[3]
print('\n\nusing given session name', session_name, '\n\n')
except IndexError:
session_name = backtesting_settings['session_name']
print('\n\nusing session name from backtesting_settings.json',
session_name, '\n\n')
symbol = backtesting_settings['symbol']
n_days = backtesting_settings['n_days']
ranges = json.load(open(os.path.join(settings_filepath, 'ranges.json')))
results_filepath = make_get_filepath(
os.path.join('backtesting_results', exchange, symbol, session_name,
''))
print(results_filepath)
settings_from_exchange_fp = results_filepath + 'settings_from_exchange.json'
if os.path.exists(settings_from_exchange_fp):
settings_from_exchange = json.load(open(settings_from_exchange_fp))
else:
tmp_live_settings = load_settings(exchange, do_print=False)
tmp_live_settings['symbol'] = backtesting_settings['symbol']
settings_from_exchange = {}
if exchange == 'binance':
bot = await create_bot_binance(user, tmp_live_settings)
settings_from_exchange['inverse'] = False
settings_from_exchange['maker_fee'] = 0.00018
settings_from_exchange['taker_fee'] = 0.00036
settings_from_exchange['exchange'] = 'binance'
elif exchange == 'bybit':
bot = await create_bot_bybit(user, tmp_live_settings)
settings_from_exchange['inverse'] = True
settings_from_exchange['maker_fee'] = -0.00025
settings_from_exchange['taker_fee'] = 0.00075
settings_from_exchange['exchange'] = 'bybit'
settings_from_exchange['min_qty'] = bot.min_qty
settings_from_exchange['min_notional'] = bot.min_notional
settings_from_exchange['qty_step'] = bot.qty_step
settings_from_exchange['price_step'] = bot.price_step
settings_from_exchange['max_leverage'] = bot.max_leverage
await bot.cc.close()
json.dump(settings_from_exchange,
open(settings_from_exchange_fp, 'w'),
indent=4)
if 'leverage' in ranges:
ranges['leverage'][1] = min(ranges['leverage'][1],
settings_from_exchange['max_leverage'])
ranges['leverage'][0] = min(ranges['leverage'][0],
ranges['leverage'][1])
backtesting_settings = {**backtesting_settings, **settings_from_exchange}
print(json.dumps(backtesting_settings, indent=4))
trades_list_filepath = os.path.join(
results_filepath, f"{n_days}_days_trades_list_cache.npy")
if os.path.exists(trades_list_filepath):
print('loading cached trade list', trades_list_filepath)
trades_list = np.load(trades_list_filepath, allow_pickle=True)
else:
agg_trades = await load_trades(exchange, user, symbol, n_days)
print('preparing trades...')
trades_list = prep_trades_list(agg_trades)
np.save(trades_list_filepath, trades_list)
jackrabbit(trades_list, backtesting_settings, ranges, results_filepath)
def jackrabbit(trades_list: [dict], backtesting_settings: dict, ranges: dict,
base_filepath: str):
if backtesting_settings['random_starting_candidate']:
best = {
key: calc_new_val((abs(ranges[key][1]) - abs(ranges[key][0])) / 2,
ranges[key], 1.0)
for key in sorted(ranges)
}
print('random starting candidate:', best)
else:
best = sort_dict_keys({k_: backtesting_settings[k_] for k_ in ranges})
n_days = backtesting_settings['n_days']
results = {}
best_gain = -9e9
candidate = best
ks = backtesting_settings['n_jackrabbit_iterations']
k = backtesting_settings['starting_k']
ms = np.array([1 / (i / 2 + 16) for i in range(ks)])
ms = ((ms - ms.min()) / (ms.max() - ms.min()))
trades_filepath = make_get_filepath(
os.path.join(base_filepath, 'trades', ''))
json.dump(backtesting_settings,
open(base_filepath + 'backtesting_settings.json', 'w'),
indent=4,
sort_keys=True)
print(backtesting_settings, '\n\n')
while k < ks - 1:
if candidate['min_markup'] >= candidate['max_markup']:
candidate['min_markup'] = candidate['max_markup']
settings_ = {**backtesting_settings, **candidate}
key = format_dict(candidate)
if key in results:
print('\nskipping', key)
candidate = get_new_candidate(ranges, best)
continue
print(f'\nk={k}, m={ms[k]:.4f} candidate:\n', candidate)
start_time = time()
trades = backtest(trades_list, settings_)
print('\ntime elapsed', round(time() - start_time, 1), 'seconds')
if not trades:
print('\nno trades')
candidate = get_new_candidate(ranges, best)
continue
k += 1
tdf = pd.DataFrame(trades).set_index('trade_id')
tdf.to_csv(trades_filepath + key + '.csv')
closest_liq = ((tdf.price - tdf.liq_price).abs() / tdf.price).min()
biggest_pos_size = tdf.pos_size.abs().max()
n_closes = len(tdf[tdf.type == 'close'])
pnl_sum = tdf.pnl.sum()
loss_sum = tdf[tdf.type == 'stop_loss'].pnl.sum()
abs_pos_sizes = tdf.pos_size.abs()
gain = (pnl_sum + settings_['balance']) / settings_['balance']
average_daily_gain = gain**(1 / n_days)
n_trades = len(tdf)
result = {
'n_closes': n_closes,
'pnl_sum': pnl_sum,
'loss_sum': loss_sum,
'average_daily_gain': average_daily_gain,
'gain': gain,
'n_trades': n_trades,
'closest_liq': closest_liq,
'biggest_pos_size': biggest_pos_size,
'n_days': n_days
}
print('\n\n', result)
results[key] = {**result, **candidate}
if gain > best_gain:
best = candidate
best_gain = gain
print('\n\n\n###############\nnew best', best,
'\naverage daily gain:', round(average_daily_gain,
5), '\n\n')
print(settings_, '\n')
print(results[key], '\n\n')
default_live_settings = load_settings(settings_['exchange'],
print_=False)
live_settings = {
k: settings_[k] if k in settings_ else default_live_settings[k]
for k in default_live_settings
}
live_settings['indicator_settings'] = {
'ema': {
'span': best['ema_span']
}
}
json.dump(live_settings,
open(base_filepath + 'best_result_live_settings.json',
'w'),
indent=4,
sort_keys=True)
json.dump(results[key],
open(base_filepath + 'best_result.json', 'w'),
indent=4,
sort_keys=True)
candidate = get_new_candidate(ranges, best, m=ms[k])
pd.DataFrame(results).T.to_csv(base_filepath + 'results.csv')
async def load_trades(exchange: str, user: str, symbol: str,
n_days: float) -> pd.DataFrame:
def skip_ids(id_, ids_):
if id_ in ids_:
print('skipping from', id_)
while id_ in ids_:
id_ -= 1
print(' to', from_id)
return id_
cc = init_ccxt(exchange, user)
try:
if exchange == 'binance':
fetch_trades_func = binance_fetch_trades
elif exchange == 'bybit':
fetch_trades_func = bybit_fetch_trades
else:
print(exchange, 'not found')
return
filepath = make_get_filepath(
f'historical_data/{exchange}/agg_trades_futures/{symbol}/')
cache_filepath = make_get_filepath(
f'historical_data/{exchange}/agg_trades_futures/{symbol}_cache/')
cache_filenames = [
f for f in os.listdir(cache_filepath) if f.endswith('.csv')
]
ids = set()
if cache_filenames:
print('loading cached trades...')
cached_trades = pd.concat(
[pd.read_csv(cache_filepath + f) for f in cache_filenames],
axis=0)
cached_trades = cached_trades.set_index('trade_id').sort_index()
cached_trades = cached_trades[~cached_trades.index.duplicated()]
ids.update(cached_trades.index)
else:
cached_trades = None
age_limit = time() - 60 * 60 * 24 * n_days
age_limit_millis = age_limit * 1000
print('age_limit', ts_to_date(age_limit))
chunk_iterator = iter_chunks(exchange, symbol)
chunk = next(chunk_iterator)
chunks = {} if chunk is None else {int(chunk.index[0]): chunk}
if chunk is not None:
ids.update(chunk.index)
min_id = min(ids) if ids else 0
new_trades = await fetch_trades_func(cc, symbol)
cached_ids = set()
k = 0
while True:
if new_trades[0]['timestamp'] <= age_limit_millis:
break
from_id = new_trades[0]['trade_id'] - 1
while True:
if chunk is None:
min_id = 0
break
from_id = skip_ids(from_id, ids)
if from_id < min_id:
chunk = next(chunk_iterator)
if chunk is None:
min_id = 0
break
else:
chunks[int(chunk.index[0])] = chunk
ids.update(chunk.index)
min_id = min(ids)
if chunk.timestamp.max() < age_limit_millis:
break
else:
break
from_id = skip_ids(from_id, ids)
from_id -= 999
new_trades = await fetch_trades_func(cc, symbol,
from_id=from_id) + new_trades
k += 1
if k % 20 == 0:
print('dumping cache')
cache_df = pd.DataFrame([
t for t in new_trades if t['trade_id'] not in cached_ids
]).set_index('trade_id')
cache_df.to_csv(cache_filepath + str(int(time() * 1000)) +
'.csv')
cached_ids.update(cache_df.index)
new_trades_df = pd.DataFrame(new_trades).set_index('trade_id')
trades_updated = pd.concat(list(chunks.values()) +
[new_trades_df, cached_trades],
axis=0)
no_dup = trades_updated[~trades_updated.index.duplicated()]
no_dup_sorted = no_dup.sort_index()
chunk_size = 100000
chunk_ids = no_dup_sorted.index // chunk_size * chunk_size
for g in no_dup_sorted.groupby(chunk_ids):
if g[0] not in chunks or len(chunks[g[0]]) != chunk_size:
print('dumping chunk', g[0])
g[1].to_csv(f'{filepath}{str(g[0])}.csv')
for f in [
f_ for f_ in os.listdir(cache_filepath) if f_.endswith('.csv')
]:
os.remove(cache_filepath + f)
await cc.close()
return no_dup_sorted[no_dup_sorted.timestamp >= age_limit_millis]
except KeyboardInterrupt:
await cc.close()
def jackrabbit(exchange: str, agg_trades: pd.DataFrame):
settings = {
'ema_spans': [10000, 38036],
'enter_long': True,
'enter_shrt': True,
'entry_amount': 0.001,
'leverage': 108,
'markup': 0.00143,
'spread': 0.00001,
'symbol': 'BTCUSDT'
}
'''
best = {'ema_spans': (39256.0, 90333.0),
'leverage': 79.0,
'markup': 0.002025,
'spread': 0.000239}
best = {"ema_spans": (45876, 67689),
"spread": -0.000149,
"leverage": 91,
"markup": 0.00093}
'''
if exchange == 'bybit':
best = {
'ema_spans': (20174, 61286),
'leverage': 86.7,
'markup': 0.0009,
'spread': 0.0
}
ranges = {
'ema_spans': (2000, 250000, 0),
'leverage': (40, 100, 1),
'markup': (0.0003, 0.006, 6),
'spread': (-0.002, 0.002, 6)
}
price_step = 0.5
inverse = True
margin_cost_limit = 0.004
maker_fee = -0.00025
taker_fee = 0.00075
settings['entry_amount'] = 1
elif exchange == 'binance':
best = {
'ema_spans': (50000, 50000),
'leverage': 90,
'markup': 0.0015,
'spread': 0.0
}
ranges = {
'ema_spans': (2000, 250000, 0),
'leverage': (40, 125, 0),
'markup': (0.0006, 0.006, 6),
'spread': (-0.002, 0.002, 6)
}
price_step = 0.01
inverse = False
margin_cost_limit = 160
maker_fee = 0.00018
taker_fee = 0.00036
settings['entry_amount'] = 0.001
else:
raise Exception(f'exchage {exchange} not found')
best = {k_: best[k_] for k_ in sorted(best)}
results = {}
best_gain = -99999
candidate = best
ks = 200
k = 0
ms = np.array([1 / (i * 0.1 + 1) for i in range(ks)])
ms = (ms - ms.min()) / (ms.max() - ms.min())
results_filename = make_get_filepath(
f'jackrabbit_results/{exchange}/{ts_to_date(time())[:19]}')
min_n_trades = len(agg_trades) / 5000
print('min_n_trades', min_n_trades)
conditions = [lambda r: r['n_trades'] > min_n_trades]
while k < ks - 1:
try:
k += 1
adf = agg_trades
key = tuple([candidate[k_] for k_ in sorted(candidate)])
if key in results:
print('skipping', key)
candidate = get_new_candidate(ranges, best)
continue
line = f'\n{k} m={ms[k]:.4f} best {tuple(best.values())}, '
line += f'candidate {tuple(candidate.values())}'
print(line)
trades, adf = backtest(adf, {
k_: candidate[k_] if k_ in candidate else settings[k_]
for k_ in settings
},
margin_cost_limit=margin_cost_limit,
maker_fee=maker_fee,
taker_fee=taker_fee,
inverse=inverse,
price_step=price_step)
if not trades:
print('\nno trades')
candidate = get_new_candidate(best)
continue
tdf = pd.DataFrame(trades).set_index('trade_id')
result = {'net_pnl': tdf.realized_pnl.sum() - tdf.fee.sum()}
result['amount_max'] = tdf.amount.max()
result['amount_min'] = tdf.amount.min()
result['amount_abs_max'] = tdf.amount.abs().max()
result['amount_abs_sum'] = tdf.amount.abs().sum()
result['n_trades'] = len(trades)
result['max_n_ddown'] = tdf.n_double_downs.max()
result['mean_n_ddown'] = tdf[
tdf.n_double_downs >= 0].n_double_downs.mean()
result['margin_cost_max'] = tdf.margin_cost.max()
result['n_liquidations'] = len(tdf[tdf.type == 'liquidation'])
result['gain'] = (result['net_pnl'] + result['margin_cost_max']) / \
result['margin_cost_max']
results[key] = result
print(f'\n{result}')
with open(results_filename + '.txt', 'a') as f:
f.write(str(key) + ' ' + str(results[key]) + '\n')
if result['gain'] > best_gain:
if all([condition(result) for condition in conditions]):
best = candidate
best_gain = result['gain']
print('\n\nnew best', best, '\n', result, '\n\n')
candidate = get_new_candidate(ranges, best, m=ms[k])
pd.DataFrame(results).to_csv(results_filename + '.csv')
except KeyboardInterrupt:
return results
return results
def jackrabbit(agg_trades: pd.DataFrame):
'''
# settings for binance
settings = {
"default_qty": 0.001,
"grid_step": 344,
"leverage": 125,
"maker_fee": 0.00018,
"margin_limit": 60,
"markups": (0.0038,),
"min_qty": 0.001,
"n_close_orders": 1,
"n_entry_orders": 7,
"price_step": 0.01,
"qty_step": 0.001,
"symbol": "BTCUSDT",
"inverse": False,
"break_on_loss": True,
}
ranges = {
'default_qty': (settings['min_qty'], settings['min_qty'] * 1, settings['qty_step']),
'grid_step': (10, 400, 1),
'markups': (0.0005, 0.005, 0.0001),
'n_close_orders': (1, 1, 1),
}
'''
# settings for bybit
settings = {
"default_qty": 1.0,
"grid_step": 344,
"leverage": 100,
"maker_fee": -0.00025,
"margin_limit": 0.001,
"markups": (0.0038, ),
"min_qty": 1.0,
"n_close_orders": 1,
"n_entry_orders": 7,
"price_step": 0.5,
"qty_step": 1.0,
"symbol": "BTCUSD",
"inverse": True,
"break_on_loss": True,
}
ranges = {
'default_qty': (1, 30, 1),
'grid_step': (1, 400, 1),
'margin_limit': (0.001, 0.001, 0.0001),
'markups': (0.0001, 0.01, 0.0001),
'n_close_orders': (1, 10, 1),
}
tweakable = {
'default_qty': 0.0,
'grid_step': 0.0,
'markups': (0.0, 0.0),
'n_close_orders': 0.0
}
best = {}
for key in tweakable:
if type(tweakable[key]) == tuple:
best[key] = tuple(
sorted([
calc_new_val((ranges[key][1] - ranges[key][0]) / 2,
ranges[key], 1.0) for _ in tweakable[key]
]))
else:
best[key] = calc_new_val((ranges[key][1] - ranges[key][0]) / 2,
ranges[key], 1.0)
# optional: uncomment to use settings as start candidate.
#best = {k_: settings[k_] for k_ in sorted(ranges)}
settings = sort_dict_keys(settings)
best = sort_dict_keys(best)
results = {}
best_gain = -99999999
candidate = best
ks = 200
k = 0
ms = np.array([1 / (i / 2 + 16) for i in range(ks)])
ms = ((ms - ms.min()) / (ms.max() - ms.min()))
results_filename = make_get_filepath(
f'jackrabbit_results_grid/{ts_to_date(time())[:19]}')
if settings['inverse']:
results_filename += '_inverse'
n_days = (agg_trades.timestamp.iloc[-1] -
agg_trades.timestamp.iloc[0]) / 1000 / 60 / 60 / 24
settings['n_days'] = n_days
print('n_days', n_days)
# conditions for result approval
conditions = [
lambda r: True,
]
df = prep_df(agg_trades)
while k < ks - 1:
try:
k += 1
key = tuple([candidate[k_] for k_ in sorted(candidate)])
if key in results:
print('skipping', key)
candidate = get_new_candidate(ranges, best)
continue
line = f'\n{k} m={ms[k]:.4f} best {tuple(best.values())}, '
line += f'candidate {tuple(candidate.values())}'
print(line)
settings_ = {
k_: candidate[k_] if k_ in candidate else settings[k_]
for k_ in sorted(settings)
}
trades = backtest(df, settings_)
if not trades:
print('\nno trades')
candidate = get_new_candidate(ranges, best)
continue
tdf = pd.DataFrame(trades).set_index('trade_id')
n_closes = len(tdf[tdf.type == 'close'])
pnl_sum = tdf.pnl.sum()
loss_sum = tdf[tdf.pnl < 0.0].pnl.sum()
abs_pos_sizes = tdf.pos_size.abs()
if settings['inverse']:
max_margin_cost = (abs_pos_sizes / tdf.pos_price /
settings_['leverage']).max()
else:
max_margin_cost = (abs_pos_sizes * tdf.pos_price /
settings_['leverage']).max()
gain = (pnl_sum +
settings_['margin_limit']) / settings_['margin_limit']
n_trades = len(tdf)
result = {
'n_closes': n_closes,
'pnl_sum': pnl_sum,
'loss_sum': loss_sum,
'max_margin_cost': max_margin_cost,
'gain': gain,
'n_trades': n_trades
}
print('\n', result)
results[key] = result
if gain > best_gain and all([c(results[key]) for c in conditions]):
best = candidate
best_gain = gain
print('\n\nnew best', best, '\n', gain, '\n')
print(settings_)
print(results[key], '\n\n')
candidate = get_new_candidate(ranges, best, m=ms[k])
pd.DataFrame(results).T.to_csv(results_filename + '.csv')
except KeyboardInterrupt:
return results
return results
