def backtest_market(entry_funcs, exit_funcs, interval, _date, smas, emas,
from_file, to_file, plot, exchange, db_client, log_level,
market):
'''
Backtests strategies for a specific market.
Args:
entry_funcs(list): list of entry functions to test.
exit_funcs(list): list of entry functions to test.
interval(string): time between measures.
_date(list): init and end point to backtest.
smas(list): list of SMA values to use.
emas(list): list of EMA values to use.
to_file(bool): plot to file.
from_file(bool): get data from file.
plot(bool): plot data.
markets(string): list with markets to backtest or empty to test all available markets.
Returns:
float: returns backtests profit & loss value for applied strategies.
'''
date = [0, 0]
total = 0
#market = check_market_name(market)
#global cached
is_cached = False
entry_points_x = []
entry_points_y = []
exit_points_x = []
exit_points_y = []
full_log = '[Market analysis]: ' + market + '\n'
if from_file:
try:
data = get_data_from_file(market, interval=interval)
except Exception as e:
log.exception(e)
log.error(f"Can\'t find {market} in files")
return 0
data_init = data
if isinstance(_date[0], str):
date[0], date[1] = time_to_index(data, _date)
else:
date = _date
if date[1] == 0:
data = data[date[0]:]
else:
data = data[date[0]:date[1]]
else:
if market in cached and \
cached[market]['interval'] == interval and \
cached[market]['init_date'] == _date[0] and \
cached[market]['end_date'] == _date[1]:
# Check if cached data is the same as you want.
data = cached[market]['data']
cached[market]['last'] = 2
is_cached = True
else:
try:
data = get_historical_data(market,
interval=interval,
init_date=_date[0],
end_date=_date[1],
exchange=exchange)
date[0], date[1] = 0, len(data)
except Exception as e:
log.exception(e)
log.error(f"Can't find {market} in BD")
return 0
#continue
data_init = data
aux_buy = False
buy_price = 0
high_price = 0
## Test for volume.
#if data.BaseVolume.mean() < 20:
# log(full_log, 1, log_level)
# del data
# del data_init
# return 0
#Tests several functions.
for i in range(len(data) - 110):
if not aux_buy:
if is_time_to_buy(data[i:i + 110], entry_funcs, smas, emas):
buy_price = data_init.Ask.iloc[i + 109 + date[0]]
high_price = buy_price
entry_points_x.append(i + 109)
entry_points_y.append(data_init.Ask.iloc[i + 109 + date[0]])
if exit_funcs:
aux_buy = True
full_log += str(data_init.time.iloc[i + 109 + date[0]]) + \
' [BUY] @ ' + str(data_init.Ask.iloc[i + 109 + date[0]]) + '\n'
else:
# Used for trailing stop loss.
if data_init.Last.iloc[i + 109 + date[0]] > high_price:
high_price = data_init.Last.iloc[i + 109 + date[0]]
if is_time_to_exit(data[i:i + 110],
exit_funcs,
smas,
emas,
stop=2,
bought_at=buy_price,
max_price=high_price):
exit_points_x.append(i + 109)
exit_points_y.append(data_init.Bid.iloc[i + 109 + date[0]])
aux_buy = False
total += round(
((data_init.Bid.iloc[i + 109 + date[0]] - buy_price) /
buy_price) * 100, 2)
full_log += str(data_init.time.iloc[i + 109 + date[0]]) + \
' [SELL]@ ' + str(data_init.Bid.iloc[i + 109 + date[0]]) + '\n'
full_log += '[P&L] > ' + str(total) + '%.' + '\n'
del data_init
# Use plot_data for just a few markets. If you try to run plot_data for several markets,
# computer can start run really slow.
try:
if plot:
plot_data(data,
name=market,
date=[0, 0],
smas=smas,
emas=emas,
entry_points=(entry_points_x, entry_points_y),
exit_points=(exit_points_x, exit_points_y),
show_smas=True,
show_emas=False,
show_bbands=False,
to_file=to_file)
except Exception as e:
log.error("Ploting data")
log.exception(e)
if not is_cached:
cached[market] = {
'interval': interval,
'init_date': _date[0],
'end_date': _date[1],
'data': data,
'last': 2
}
#if len(exit_points_x):
# log(market + ' > ' + str(total), log_level)
log.info(f"[{market}][TOTAL]> {total:.2}")
log.info(full_log)
if isnan(total):
log.error("Total is isnan")
return 0
return total
def backtest(markets,
entry_funcs,
exit_funcs=[],
_date=[0, 0],
smas=var.default_smas,
emas=var.default_emas,
interval=var.default_interval,
plot=False,
to_file=True,
from_file=False,
exchange="bittrex",
base_market='BTC',
log_level=1,
mp_level="medium"):
'''
Backtests strategies.
Args:
markets(list): list with markets to backtest or empty to test all available markets.
entry_funcs(list): list of entry functions to test.
exit_funcs(list): list of entry functions to test.
_date(list): init and end point to backtest.
Ex: '1-1-2017 11:10'
smas(list): list of SMA values to use.
emas(list): list of EMA values to use.
interval(string): time between measures.
plot(bool): plot data.
to_file(bool): plot to file.
from_file(bool): get data from file.
base_market(string): base market to use.
log_level(int): log level - 0-2.
mp_level(string): multiprocessing level - [low, medium, high].
Returns:
bool: returns True in case of success.
log_level:
0 - Only presents total.
1 - Writes logs to file.
2 - Writes logs to file and prints on screen.
Default is 2.
'''
signal.signal(signal.SIGINT, signal_handler)
#global cached
if not from_file:
# Connects to DB.
try:
db_client = connect_db()
except Exception as e:
log.exception(e)
sys.exit(1)
else:
db_client = 0
# For all markets.
if not len(markets):
y = 'y' #raw_input("Want to run all markets? ")
if y == 'y':
if from_file:
markets = get_markets_on_files(interval, base=base_market)
else:
markets = get_markets_list(base_market, exchange)
else:
log.error("Without files to analyse.")
# Prevents errors from markets and funcs as str.
if not isinstance(markets, list): markets = [markets]
if not isinstance(entry_funcs, list): entry_funcs = [entry_funcs]
if not isinstance(exit_funcs, list): exit_funcs = [exit_funcs]
# For selected markets.
if from_file:
markets = manage_files(markets, interval=interval)
log.info(str(len(markets)) + " files/chunks to analyse...")
# Create a multiprocessing Pool
pool = Pool(num_processors(mp_level))
# Display information about pool.
total = pool.map(
partial(backtest_market, entry_funcs, exit_funcs, interval, _date,
smas, emas, from_file, to_file, plot, exchange, db_client,
log_level), markets)
pool.close()
pool.join()
log.info(" Total > " + str(sum(total)))
for k in cached.keys():
if cached[k]['last'] < 1:
del cached[k]
else:
cached[k]['last'] = cached[k]['last'] - 1
return sum(total)
def signal_handler(sig, frame):
log.info(f"You pressed Ctrl+C!")
sys.exit(0)
def tick_by_tick(market,
entry_funcs,
exit_funcs,
interval=var.default_interval,
smas=var.default_smas,
emas=var.default_emas,
refresh_interval=1,
from_file=True,
plot=False,
log_level=2):
'''
Simulates a working bot, in realtime or in faster speed,
using pre own data from DB or file,
to test an autonomous bot on a specific market.
Args:
markets(string): list with markets to backtest or
empty to run all available markets.
entry_funcs(list): list of entry functions to test.
exit_funcs(list): list of entry functions to test.
interval(string): time between measures.
smas(list): list of SMA values to use.
emas(list): list of EMA values to use.
refresh_interval(int): Refresh rate.
main_coins(list):
'''
#log_level:
# 0 - Only presents total.
# 1 - Writes logs to file.
# 2 - Writes logs to file and prints on screen.
# Default is 2.
#plt.ion()
var.global_log_level = log_level
signal.signal(signal.SIGINT, signal_handler)
date = [0, 0]
total = 0
#market = check_market_name(market)
entry_points_x = []
entry_points_y = []
exit_points_x = []
exit_points_y = []
if not isinstance(entry_funcs, list): entry_funcs = [entry_funcs]
if not isinstance(exit_funcs, list): exit_funcs = [exit_funcs]
log.info(f"[Market analysis]: {market}")
if from_file:
try:
data = get_data_from_file(market, interval=interval)
except Exception as e:
log.error(f"Unable to get data from file: {e}")
log.error(f"Unable to find {market} in files.")
return 0
data_init = data
#if type(_date[0]) is str:
# date[0], date[1] = time_to_index(data, _date)
#if date[1] == 0:
# data = data[date[0]:]
#else:
# data = data[date[0]:date[1]]
else:
try:
data = get_historical_data(market,
interval=interval,
init_date=_date[0],
end_date=_date[1])
date[0], date[1] = 0, len(data)
data_init = data
except Exception as e:
log.error(f"Unable to get data from file: {e}")
log.error(f"Unable to find {market} in DB.")
return 0
aux_buy = False
buy_price = 0
high_price = 0
#plt.show()
#Tests several functions.
for i in range(len(data) - 110):
start_time = time()
#print(data_init.Last.iloc[i])
if not aux_buy:
if is_time_to_buy(data[i:i + 110], entry_funcs, smas, emas):
buy_price = data_init.Ask.iloc[i + 109 + date[0]]
high_price = buy_price
entry_points_x.append(i + 109)
entry_points_y.append(data_init.Ask.iloc[i + 109 + date[0]])
if exit_funcs:
aux_buy = True
log.info(f'''{data_init.time.iloc[i + 109 + date[0]]} \
[BUY] @ {data_init.Ask.iloc[i + 109 + date[0]]}''')
else:
# Used for trailing stop loss.
if data_init.Last.iloc[i + 109 + date[0]] > high_price:
high_price = data_init.Last.iloc[i + 109 + date[0]]
if is_time_to_exit(data[i:i + 110],
exit_funcs,
smas,
emas,
stop=2,
bought_at=buy_price,
max_price=high_price):
exit_points_x.append(i + 109)
exit_points_y.append(data_init.Bid.iloc[i + 109 + date[0]])
aux_buy = False
total += round(
((data_init.Bid.iloc[i + 109 + date[0]] - buy_price) /
buy_price) * 100, 2)
log.info(f'''{data_init.time.iloc[i + 109 + date[0]]} \
[SELL]@ {data_init.Bid.iloc[i + 109 + date[0]]}''')
log.info(f'[P&L] > {total}%.')
#plt.plot(data.Last.iloc[i:i+50])
#plt.draw()
#plt.clf()
# In case of processing time is bigger than *refresh_interval* doesn't sleep.
if refresh_interval - (time() - start_time) >= 0:
sleep(refresh_interval - (time() - start_time))
return total
def realtime(exchanges,
entry_funcs,
exit_funcs,
interval=var.default_interval,
smas=var.default_smas,
emas=var.default_volume_emas,
refresh_interval=10,
simulation=True,
main_coins=("BTC", "USDT"),
log_level=1):
'''
Bot using realtime data, doesn't need DB or csv files to work.
Args:
exchanges(list): list of exchanges.
entry_funcs(list): list of entry functions to test.
exit_funcs(list): list of entry functions to test.
markets(string): list with markets to backtest or empty to run all available markets.
interval(string): time between measures.
smas(list): list of SMA values to use.
emas(list): list of EMA values to use.
refresh_interval(int): Data refresh rate.
simulation(bool): Defines if it's running as a simulation or real money mode.
main_coins(tuple): tuple of main coins.
'''
#log_level:
# 0 - Only presents total.
# 1 - Writes logs to file.
# 2 - Writes logs to file and prints on screen.
# Default is 2.
#var.global_log_level = log_level
signal.signal(signal.SIGINT, signal_handler)
validate = smas[-1] + 5
nr_exchanges = len(exchanges)
if not isinstance(exchanges, list): exchanges = [exchanges]
if not isinstance(entry_funcs, list): entry_funcs = [entry_funcs]
if not isinstance(exit_funcs, list): exit_funcs = [exit_funcs]
portfolio = {} # Owned coins list.
coins = {}
# Bittrex exchange
if "bittrex" in exchanges:
if simulation:
bt = Bittrex('', '')
log.debug(f"Starting Bot with Bittrex")
else:
try:
bt = RiskManagement(var.ky, var.sct)
except Exception as e:
log.error(f"Unable to connect to Bittrex: {e}")
nr_exchanges -= 1
# Binance exchange
if "binance" in exchanges:
if simulation:
try:
bnb = Binance('', '')
log.info("Starting Bot with Binance")
except Exception as e:
log.error(f"Unable to connect to Binance: {e}")
else:
#print(f"Can't use Binance exchange in real scenario, just simulation.")
#sys.exit(1)
try:
bnb = Bnb(var.bnb_ky, var.bnb_sct)
except Exception as e:
log.error(f"Unable to connect to Binance")
nr_exchanges -= 1
if not nr_exchanges:
sys.exit(1)
markets = []
while True:
start_time = time()
if "bittrex" in exchanges:
markets += bt.get_market_summaries()['result']
if "binance" in exchanges: markets += bnb.get_ticker()
for market in markets:
# Needed to pass unicode to string.
# Binance
if 'MarketName' in market:
market_name = 'BT_' + str(market['MarketName'])
elif 'symbol' in market:
market = binance2btrx(market)
market_name = 'BN_' + market['MarketName']
# Checks if pair is included in main coins.
if (market_name.startswith('BT_') and market_name.split('-')[0]) or \
(market_name.startswith('BN_') and market_name.endswith(main_coins)):
# Checks if market already exists in analysed coins.
if market_name in coins:
# Checks if has enough data to analyse.
if coins[market_name] == validate:
locals()[market_name] = pd.DataFrame.append(
locals()[market_name], [market]).tail(validate)
# If not, adds data and keep going.
else:
locals()[market_name] = pd.DataFrame.append(
locals()[market_name], [market])
coins[market_name] += 1
continue
# If not adds coin to system.
else:
locals()[market_name] = pd.DataFrame([market])
coins[market_name] = 1
continue
if '-' in market_name:
# Renames OpenBuy and OpenSell in Bittrex
data = locals()[market_name].rename(index=str,
columns={
"OpenBuyOrders":
"OpenBuy",
"OpenSellOrders":
"OpenSell"
})
else:
data = locals()[market_name]
# Checks if coin is in portfolio and looks for a sell opportunity.
if market_name in portfolio:
# Needed to make use of stop loss and trailing stop loss functions.
if portfolio[market_name]['max_price'] < data.Bid.iloc[-1]:
portfolio[market_name]['max_price'] = data.Bid.iloc[-1]
if is_time_to_exit(
data,
exit_funcs,
bought_at=portfolio[market_name]['bought_at'],
max_price=portfolio[market_name]['max_price'],
count=portfolio[market_name]['count']):
# implementar binance
if not simulation:
#REAL
sell_res = bt.sell(
market_name,
portfolio[market_name]['quantity'],
data.Bid.iloc[-1])
# M U D A R
sold_at = sell_res
log.info(f'[SELL]@ {sold_at} > {market_name}')
res = ((sold_at - portfolio[market_name]['bought_at'])/\
portfolio[market_name]['bought_at'])*100
log.info(f'[P&L] {market_name}> {res:.2f}%.')
# implementar binance
else:
#SIMULATION
#log.info('> https://bittrex.com/Market/Index?MarketName=' + market_name)
log.info(
f'[SELL]@ {data.Bid.iloc[-1]} > {market_name}')
res = ((data.Bid.iloc[-1] - portfolio[market_name]['bought_at'])/\
portfolio[market_name]['bought_at'])*100
log.info(f'[P&L] {market_name} > {res:.2f}%.')
locals()[market_name].to_csv(
f"df_{market_name}{data.time.iloc[-1]}.csv")
del locals()[market_name]
del portfolio[market_name]
coins.pop(market_name)
#if not time to exit, increment count.
else:
portfolio[market_name]['count'] += 1
# if the coin is not on portfolio, checks if is time to buy.
else:
if is_time_to_buy(data, entry_funcs):
if not simulation:
#REAL
sucs, msg = bt.buy(market,
data.Ask.iloc[-1] * 1.01)
if sucs:
portfolio[market_name] = {}
portfolio[market_name]['bought_at'] = msg[0]
portfolio[market_name]['max_price'] = msg[0]
portfolio[market_name]['quantity'] = msg[1]
portfolio[market_name]['count'] = 0
log.info(f'[BUY]@ {msg[0]} > {market_name}')
else:
log.info(
f"[XXXX] Could not buy @ {data.Ask.iloc[-1] * 1.01} \
[MSG>] {msg}")
else:
#SIMULATION
portfolio[market_name] = {}
portfolio[market_name][
'bought_at'] = data.Ask.iloc[-1]
portfolio[market_name][
'max_price'] = data.Ask.iloc[-1]
portfolio[market_name]['quantity'] = 1
portfolio[market_name]['count'] = 0
#log.info('> https://bittrex.com/Market/Index?MarketName='+market_name)
log.info(
f'[BUY]@{data.Ask.iloc[-1]} > {market_name}')
# In case of processing time is bigger than *refresh_interval* doesn't sleep.
if refresh_interval - (time() - start_time) > 0:
sleep(refresh_interval - (time() - start_time))
