async def main():
parser = argparse.ArgumentParser(
prog="Downloader", description="Download ticks from exchange API.")
parser.add_argument(
"-d",
"--download-only",
help="download only, do not dump ticks caches",
action="store_true",
)
parser.add_argument(
"-oh",
"--ohlcv",
help="use 1m ohlcv instead of 1s ticks",
action="store_true",
)
parser = add_argparse_args(parser)
args = parser.parse_args()
config = await prepare_backtest_config(args)
if args.ohlcv:
data = load_hlc_cache(config["symbol"], config["start_date"],
config["end_date"])
else:
downloader = Downloader(config)
await downloader.download_ticks()
if not args.download_only:
await downloader.prepare_files()
async def main():
parser = argparse.ArgumentParser(prog='Optimize',
description='Optimize passivbot config.')
parser.add_argument('-o',
'--optimize_config',
type=str,
required=False,
dest='optimize_config_path',
default='configs/optimize/default.hjson',
help='optimize config hjson file')
parser.add_argument(
'-t',
'--start',
type=str,
required=False,
dest='starting_configs',
default=None,
help=
'start with given live configs. single json file or dir with multiple json files'
)
parser.add_argument('-i',
'--iters',
type=int,
required=False,
dest='iters',
default=None,
help='n optimize iters')
parser = add_argparse_args(parser)
args = parser.parse_args()
config = await prepare_optimize_config(args)
await execute_optimize(config)
async def main():
parser = argparse.ArgumentParser(prog='Backtest', description='Backtest given passivbot config.')
parser.add_argument('live_config_path', type=str, help='path to live config to test')
parser = add_argparse_args(parser)
parser.add_argument(
"-lw",
"--long_wallet_exposure_limit",
"--long-wallet-exposure-limit",
type=float,
required=False,
dest="long_wallet_exposure_limit",
default=None,
help="specify long wallet exposure limit, overriding value from live config",
)
parser.add_argument(
"-sw",
"--short_wallet_exposure_limit",
"--short-wallet-exposure-limit",
type=float,
required=False,
dest="short_wallet_exposure_limit",
default=None,
help="specify short wallet exposure limit, overriding value from live config",
)
args = parser.parse_args()
config = await prepare_backtest_config(args)
live_config = load_live_config(args.live_config_path)
config.update(live_config)
if args.long_wallet_exposure_limit is not None:
print(
f"overriding long wallet exposure limit ({config['long']['pbr_limit']}) "
f"with new value: {args.long_wallet_exposure_limit}"
)
config["long"]["pbr_limit"] = args.long_wallet_exposure_limit
if args.short_wallet_exposure_limit is not None:
print(
f"overriding short wallet exposure limit ({config['shrt']['pbr_limit']}) "
f"with new value: {args.short_wallet_exposure_limit}"
)
config["shrt"]["pbr_limit"] = args.short_wallet_exposure_limit
if 'spot' in config['market_type']:
live_config = spotify_config(live_config)
downloader = Downloader(config)
print()
for k in (keys := ['exchange', 'spot', 'symbol', 'market_type', 'config_type', 'starting_balance', 'start_date', 'end_date',
'latency_simulation_ms']):
if k in config:
print(f"{k: <{max(map(len, keys)) + 2}} {config[k]}")
async def main():
parser = argparse.ArgumentParser(
prog='Downloader', description='Download ticks from exchange API.')
parser.add_argument('-d',
'--download-only',
help='download only, do not dump ticks caches',
action='store_true')
parser = add_argparse_args(parser)
args = parser.parse_args()
config = await prepare_backtest_config(args)
downloader = Downloader(config)
await downloader.download_ticks()
if not args.download_only:
await downloader.prepare_files()
async def main():
parser = argparse.ArgumentParser(prog='Optimize', description='Optimize passivbot config.')
parser = add_argparse_args(parser)
parser.add_argument('-t', '--start', type=str, required=False, dest='starting_configs',
default=None,
help='start with given live configs. single json file or dir with multiple json files')
args = parser.parse_args()
for config in await prep_config(args):
try:
template_live_config = get_template_live_config(config['n_spans'])
config = {**template_live_config, **config}
dl = Downloader(config)
data = await dl.get_sampled_ticks()
shm = shared_memory.SharedMemory(create=True, size=data.nbytes)
shdata = np.ndarray(data.shape, dtype=data.dtype, buffer=shm.buf)
shdata[:] = data
del data
config['n_days'] = (shdata[-1][0] - shdata[0][0]) / (1000 * 60 * 60 * 24)
config['optimize_dirpath'] = make_get_filepath(os.path.join(config['optimize_dirpath'],
ts_to_date(time())[:19].replace(':', ''), ''))
print()
for k in (keys := ['exchange', 'symbol', 'starting_balance', 'start_date', 'end_date', 'latency_simulation_ms',
'do_long', 'do_shrt', 'minimum_bankruptcy_distance', 'maximum_hrs_no_fills',
'maximum_hrs_no_fills_same_side', 'iters', 'n_particles', 'sliding_window_size',
'n_spans']):
if k in config:
print(f"{k: <{max(map(len, keys)) + 2}} {config[k]}")
print()
backtest_wrap = BacktestWrap(shdata, config)
post_processing = PostProcessing()
if config['starting_configs']:
starting_configs = get_starting_configs(config)
initial_positions = [backtest_wrap.config_to_xs(cfg) for cfg in starting_configs]
else:
initial_positions = []
pso_multiprocess(backtest_wrap.rf,
config['n_particles'],
backtest_wrap.bounds,
config['options']['c1'],
config['options']['c2'],
config['options']['w'],
n_cpus=config['num_cpus'],
iters=config['iters'],
initial_positions=initial_positions,
post_processing_func=post_processing.process)
finally:
async def main():
logging.basicConfig(format="", level=os.environ.get("LOGLEVEL", "INFO"))
parser = argparse.ArgumentParser(
prog="Optimize multi symbol",
description="Optimize passivbot config multi symbol")
parser.add_argument(
"-o",
"--optimize_config",
type=str,
required=False,
dest="optimize_config_path",
default="configs/optimize/harmony_search.hjson",
help="optimize config hjson file",
)
parser.add_argument(
"-t",
"--start",
type=str,
required=False,
dest="starting_configs",
default=None,
help=
"start with given live configs. single json file or dir with multiple json files",
)
parser.add_argument("-i",
"--iters",
type=int,
required=False,
dest="iters",
default=None,
help="n optimize iters")
parser.add_argument("-c",
"--n_cpus",
type=int,
required=False,
dest="n_cpus",
default=None,
help="n cpus")
parser.add_argument(
"-le",
"--long",
type=str,
required=False,
dest="long_enabled",
default=None,
help="long enabled: [y/n]",
)
parser.add_argument(
"-se",
"--short",
type=str,
required=False,
dest="short_enabled",
default=None,
help="short enabled: [y/n]",
)
parser.add_argument(
"-pm",
"--passivbot_mode",
"--passivbot-mode",
type=str,
required=False,
dest="passivbot_mode",
default=None,
help="passivbot mode options: [s/static_grid, r/recursive_grid]",
)
parser.add_argument(
"-sf",
"--score_formula",
"--score-formula",
type=str,
required=False,
dest="score_formula",
default=None,
help=
"passivbot score formula options: [adg_PAD_mean, adg_PAD_std, adg_DGstd_ratio, adg_mean, adg_min, adg_PAD_std_min]",
)
parser.add_argument(
"-oh",
"--ohlcv",
help="use 1m ohlcv instead of 1s ticks",
action="store_true",
)
parser = add_argparse_args(parser)
args = parser.parse_args()
args.symbol = "BTCUSDT" # dummy symbol
config = await prepare_optimize_config(args)
if args.score_formula is not None:
if args.score_formula not in [
"adg_PAD_mean",
"adg_PAD_std",
"adg_DGstd_ratio",
"adg_mean",
"adg_min",
"adg_PAD_std_min",
]:
logging.error(f"unknown score formula {args.score_formula}")
logging.error(f"using score formula {config['score_formula']}")
else:
config["score_formula"] = args.score_formula
if args.passivbot_mode is not None:
if args.passivbot_mode in ["s", "static_grid", "static"]:
config["passivbot_mode"] = "static_grid"
elif args.passivbot_mode in ["r", "recursive_grid", "recursive"]:
config["passivbot_mode"] = "recursive_grid"
else:
raise Exception(f"unknown passivbot mode {args.passivbot_mode}")
passivbot_mode = config["passivbot_mode"]
assert passivbot_mode in [
"recursive_grid",
"static_grid",
], f"unknown passivbot mode {passivbot_mode}"
config.update(get_template_live_config(passivbot_mode))
config["exchange"], _, _ = load_exchange_key_secret(config["user"])
args = parser.parse_args()
if args.long_enabled is None:
config["long"]["enabled"] = config["do_long"]
else:
if "y" in args.long_enabled.lower():
config["long"]["enabled"] = config["do_long"] = True
elif "n" in args.long_enabled.lower():
config["long"]["enabled"] = config["do_long"] = False
else:
raise Exception("please specify y/n with kwarg -le/--long")
if args.short_enabled is None:
config["short"]["enabled"] = config["do_short"]
else:
if "y" in args.short_enabled.lower():
config["short"]["enabled"] = config["do_short"] = True
elif "n" in args.short_enabled.lower():
config["short"]["enabled"] = config["do_short"] = False
else:
raise Exception("please specify y/n with kwarg -le/--short")
if args.symbol is not None:
config["symbols"] = args.symbol.split(",")
if args.n_cpus is not None:
config["n_cpus"] = args.n_cpus
config["ohlcv"] = args.ohlcv
print()
lines = [(k, getattr(args, k)) for k in args.__dict__
if args.__dict__[k] is not None]
for line in lines:
logging.info(
f"{line[0]: <{max([len(x[0]) for x in lines]) + 2}} {line[1]}")
print()
# download ticks .npy file if missing
if config["ohlcv"]:
cache_fname = f"{config['start_date']}_{config['end_date']}_ohlcv_cache.npy"
else:
cache_fname = f"{config['start_date']}_{config['end_date']}_ticks_cache.npy"
exchange_name = config["exchange"] + ("_spot" if config["market_type"]
== "spot" else "")
config["symbols"] = sorted(config["symbols"])
for symbol in config["symbols"]:
cache_dirpath = f"backtests/{exchange_name}/{symbol}/caches/"
if not os.path.exists(cache_dirpath +
cache_fname) or not os.path.exists(
cache_dirpath +
"market_specific_settings.json"):
logging.info(f"fetching data {symbol}")
args.symbol = symbol
if config["ohlcv"]:
data = load_hlc_cache(
symbol,
config["start_date"],
config["end_date"],
base_dir=config["base_dir"],
spot=config["spot"],
)
else:
tmp_cfg = await prepare_backtest_config(args)
downloader = Downloader({**config, **tmp_cfg})
await downloader.get_sampled_ticks()
# prepare starting configs
cfgs = []
if args.starting_configs is not None:
logging.info("preparing starting configs...")
if os.path.isdir(args.starting_configs):
for fname in os.listdir(args.starting_configs):
try:
cfg = load_live_config(
os.path.join(args.starting_configs, fname))
assert determine_passivbot_mode(
cfg) == passivbot_mode, "wrong passivbot mode"
cfgs.append(cfg)
except Exception as e:
logging.error(f"error loading config {fname}: {e}")
elif os.path.exists(args.starting_configs):
hm_load_failed = True
if "hm_" in args.starting_configs:
try:
hm = json.load(open(args.starting_configs))
for k in hm:
cfg = {
"long": hm[k]["long"]["config"],
"short": hm[k]["short"]["config"]
}
assert (determine_passivbot_mode(cfg) == passivbot_mode
), "wrong passivbot mode in harmony memory"
cfgs.append(cfg)
logging.info(
f"loaded harmony memory {args.starting_configs}")
hm_load_failed = False
except Exception as e:
logging.error(
f"error loading harmony memory {args.starting_configs}: {e}"
)
if hm_load_failed:
try:
cfg = load_live_config(args.starting_configs)
assert determine_passivbot_mode(
cfg) == passivbot_mode, "wrong passivbot mode"
cfgs.append(cfg)
except Exception as e:
logging.error(
f"error loading config {args.starting_configs}: {e}")
config["starting_configs"] = cfgs
harmony_search = HarmonySearch(config)
harmony_search.run()
async def main():
parser = argparse.ArgumentParser(
prog="Backtest", description="Backtest given passivbot config.")
parser.add_argument("live_config_path",
type=str,
help="path to live config to test")
parser = add_argparse_args(parser)
parser.add_argument(
"-lw",
"--long_wallet_exposure_limit",
"--long-wallet-exposure-limit",
type=float,
required=False,
dest="long_wallet_exposure_limit",
default=None,
help=
"specify long wallet exposure limit, overriding value from live config",
)
parser.add_argument(
"-sw",
"--short_wallet_exposure_limit",
"--short-wallet-exposure-limit",
type=float,
required=False,
dest="short_wallet_exposure_limit",
default=None,
help=
"specify short wallet exposure limit, overriding value from live config",
)
parser.add_argument(
"-le",
"--long_enabled",
"--long-enabled",
type=str,
required=False,
dest="long_enabled",
default=None,
help="specify long enabled [y/n], overriding value from live config",
)
parser.add_argument(
"-se",
"--short_enabled",
"--short-enabled",
type=str,
required=False,
dest="short_enabled",
default=None,
help="specify short enabled [y/n], overriding value from live config",
)
parser.add_argument(
"-np",
"--n_parts",
"--n-parts",
type=int,
required=False,
dest="n_parts",
default=None,
help="set n backtest slices to plot",
)
parser.add_argument(
"-oh",
"--ohlcv",
help="use 1m ohlcv instead of 1s ticks",
action="store_true",
)
args = parser.parse_args()
if args.symbol is None:
tmp_cfg = load_hjson_config(args.backtest_config_path)
symbols = (tmp_cfg["symbol"] if type(tmp_cfg["symbol"]) == list else
tmp_cfg["symbol"].split(","))
else:
symbols = args.symbol.split(",")
for symbol in symbols:
args = parser.parse_args()
args.symbol = symbol
config = await prepare_backtest_config(args)
config["n_parts"] = args.n_parts
live_config = load_live_config(args.live_config_path)
config.update(live_config)
if args.long_wallet_exposure_limit is not None:
print(
f"overriding long wallet exposure limit ({config['long']['wallet_exposure_limit']}) "
f"with new value: {args.long_wallet_exposure_limit}")
config["long"][
"wallet_exposure_limit"] = args.long_wallet_exposure_limit
if args.short_wallet_exposure_limit is not None:
print(
f"overriding short wallet exposure limit ({config['short']['wallet_exposure_limit']}) "
f"with new value: {args.short_wallet_exposure_limit}")
config["short"][
"wallet_exposure_limit"] = args.short_wallet_exposure_limit
if args.long_enabled is not None:
config["long"]["enabled"] = "y" in args.long_enabled.lower()
if args.short_enabled is not None:
config["short"]["enabled"] = "y" in args.short_enabled.lower()
if "spot" in config["market_type"]:
live_config = spotify_config(live_config)
config["ohlcv"] = args.ohlcv
print()
for k in (keys := [
"exchange",
"spot",
"symbol",
"market_type",
"passivbot_mode",
"config_type",
"starting_balance",
"start_date",
"end_date",
"latency_simulation_ms",
"base_dir",
]):
if k in config:
print(f"{k: <{max(map(len, keys)) + 2}} {config[k]}")
print()
if config["ohlcv"]:
data = load_hlc_cache(
symbol,
config["start_date"],
config["end_date"],
base_dir=config["base_dir"],
spot=config["spot"],
)
else:
downloader = Downloader(config)
data = await downloader.get_sampled_ticks()
config["n_days"] = round_(
(data[-1][0] - data[0][0]) / (1000 * 60 * 60 * 24), 0.1)
pprint.pprint(denumpyize(live_config))
plot_wrap(config, data)
async def main():
parser = argparse.ArgumentParser(prog='Optimize multi symbol', description='Optimize passivbot config multi symbol')
parser.add_argument('-o', '--optimize_config', type=str, required=False, dest='optimize_config_path',
default='configs/optimize/multi_symbol.hjson', help='optimize config hjson file')
parser.add_argument('-t', '--start', type=str, required=False, dest='starting_configs',
default=None,
help='start with given live configs. single json file or dir with multiple json files')
parser.add_argument('-i', '--iters', type=int, required=False, dest='iters', default=None, help='n optimize iters')
parser = add_argparse_args(parser)
args = parser.parse_args()
args.symbol = 'BTCUSDT' # dummy symbol
config = await prepare_optimize_config(args)
config.update(get_template_live_config())
config['exchange'], _, _ = load_exchange_key_secret(config['user'])
config['long']['enabled'] = config['do_long']
config['shrt']['enabled'] = config['do_shrt']
if config['long']['enabled']:
if config['shrt']['enabled']:
print('optimizing both long and short')
config['side'] = 'both'
else:
print('optimizing long')
config['side'] = 'long'
elif config['shrt']['enabled']:
print('optimizing short')
config['side'] = 'shrt'
else:
raise Exception('long, shrt or both must be enabled')
# download ticks .npy file if missing
cache_fname = f"{config['start_date']}_{config['end_date']}_ticks_cache.npy"
exchange_name = config['exchange'] + ('_spot' if config['market_type'] == 'spot' else '')
for symbol in sorted(config['symbols']):
cache_dirpath = f"backtests/{exchange_name}/{symbol}/caches/"
if not os.path.exists(cache_dirpath + cache_fname) or not os.path.exists(cache_dirpath + 'market_specific_settings.json'):
print(f'fetching data {symbol}')
args.symbol = symbol
tmp_cfg = await prepare_backtest_config(args)
downloader = Downloader({**config, **tmp_cfg})
await downloader.get_sampled_ticks()
pool = Pool(processes=config['n_cpus'])
func_wrap = FuncWrap(pool, config)
cfgs = []
if args.starting_configs is not None:
if os.path.isdir(args.starting_configs):
cfgs = []
for fname in os.listdir(args.starting_configs):
try:
cfgs.append(load_live_config(os.path.join(args.starting_configs, fname)))
except Exception as e:
print('error loading config:', e)
elif os.path.exists(args.starting_configs):
try:
cfgs = [load_live_config(args.starting_configs)]
except Exception as e:
print('error loading config:', e)
starting_xs = [func_wrap.config_to_xs(cfg) for cfg in cfgs]
n_harmonies = config['n_harmonies']
hm_considering_rate = config['hm_considering_rate']
bandwidth = config['bandwidth']
pitch_adjusting_rate = config['pitch_adjusting_rate']
iters = config['iters']
best_harmony = harmony_search(func_wrap.func, func_wrap.bounds, n_harmonies,
hm_considering_rate, bandwidth, pitch_adjusting_rate, iters,
starting_xs=starting_xs,
post_processing_func=func_wrap.post_processing_func)
best_conf = func_wrap.xs_to_config(best_harmony)
print('best conf')
print(best_conf)
return