def populate_historical(market, start_time):
binance_api = config['binance']['api']
binance_secret = config['binance']['secret']
binance_client = BinanceClient(binance_api, binance_secret)
# Get historical aggregated trade data as generator object and count number of historical trades
historical_trades = binance_client.aggregate_trade_iter(
symbol=market, start_str=start_time)
logger.info('Counting historical trades for database population.')
trade_count = sum(1 for trade in historical_trades)
logger.debug('trade_count: ' + str(trade_count))
# Get historical aggregated trade data again to refresh generator object (May make total count off by few trades)
historical_trades = binance_client.aggregate_trade_iter(
symbol=market, start_str=start_time)
count = 0
for trade in historical_trades:
process_result = process_message(trade, populate=True, symbol=market)
if process_result == False:
logger.info('Database population complete.')
break
else:
count += 1
logger.info('Processed ' + str(count) + ' of ~' +
str(trade_count) + ' historical trades.')
def populate_historical(exchange, market, start_time):
if exchange == 'binance':
binance_api = config['binance']['api']
binance_secret = config['binance']['secret']
binance_client = BinanceClient(binance_api, binance_secret)
# Get historical aggregated trade data as generator object and count number of historical trades
historical_trades = binance_client.aggregate_trade_iter(
symbol=market, start_str=start_time)
logger.info('Counting historical trades for database population.')
trade_count = sum(1 for trade in historical_trades)
logger.debug('trade_count: ' + str(trade_count))
# Get historical aggregated trade data again to refresh generator object (May make total count off by few trades)
historical_trades = binance_client.aggregate_trade_iter(
symbol=market, start_str=start_time)
count = 0
for trade in historical_trades:
process_result = process_message(trade,
populate=True,
exchange=exchange,
market=market)
if process_result == False:
logger.info('Database population complete.')
break
#logger.info('Trade document already present in database.')
else:
count += 1
completion_percentage = "{:.2f}".format(
(count / trade_count) * 100)
logger.info('Processed ' + str(count) + ' of ~' +
str(trade_count) + ' historical trades. [' +
completion_percentage + '%]')
elif exchange == 'poloniex':
logger.warning('POLONIEX DATABASE POPULATION NOT YET IMPLEMENTED.')
else:
logger.error(
'Unrecognized exchange passed to populate_historical(). Exiting.')
sys.exit(1)
sys.exit(1)
else:
logger.info('Selected Binance market ' + user_market + '.')
if backtest_duration == None:
backtest_duration = input(
'Input length of time for trade data backtesting/analysis (ex. 30 seconds/9 minutes/3 hours/2 days/1 week): '
)
test_duration = backtest_duration + ' ago UTC'
logger.debug('test_duration: ' + test_duration)
try:
logger.debug(
'Testing user-provided historical data population input.')
historical_trades = binance_client.aggregate_trade_iter(
symbol=user_market, start_str=test_duration)
logger.debug('Attempting count of trades in generator object.')
trade_count = sum(1 for trade in historical_trades)
populate_duration = test_duration
logger.debug('populate_duration: ' + populate_duration)
except:
logger.error(
'Invalid input for start of historical data population. Exiting.'
)
sys.exit(1)
if user_market == None or backtest_duration == None:
logger.error('Failed to gather valid user input. Exiting.')
sys.exit(1)
## Delete existing data for market from database ##
from binance.client import Client
from binance.enums import *
import matplotlib.pyplot as plt
import numpy as np
import csv
import pandas as pd
import seaborn as sns
import datetime
# getting recent trades
client = Client(
"wDPuqBa0zeqQRHE26takNQ5G9jLyyFkWxisrheBqmxHDhz4RPcNzR8bPLg4E4Gka",
"NwUcys7q7NLPl1ISoIAB97wyAjGNOc2Kw3nJqRJ8BwzFO4ERAsCqdOnK1eDzaPQk")
agg_trades = client.aggregate_trade_iter(symbol='ETHBTC',
start_str='15 seconds ago UTC')
# only extracting the date and price
trades = []
for t in agg_trades:
t = [t['T'], t['p']]
t[0] = datetime.datetime.utcfromtimestamp(round(t[0] / 1000))
split_t = str(t[0]).split(' ')
t[0] = split_t[1]
trades.append(t)
# creating a csv file with the trades data
with open('recent_trades.csv', 'w', newline='') as csvfile:
filewriter = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
# Get candle data(aka kline, histo)
klines = client.get_historical_klines("BNBBTC", Client.KLINE_INTERVAL_1MINUTE,
"1 day ago UTC")
df = pd.DataFrame(klines)
columns = [
'OpenTime', 'Open', 'High', 'Low', 'Close', 'Volume', 'CloseTime',
'QuoteVolume', 'NumberTrades', 'BaseVolumne', 'QuoteVolumne', 'Ignored'
]
df.columns = columns
df['OpenTime'] = pd.to_datetime(df['OpenTime'], unit='ms')
df['CloseTime'] = pd.to_datetime(df['CloseTime'], unit='ms')
df.to_csv('klines_data.csv', index=False)
# Get transactions(aka trades)
agg_trades = client.aggregate_trade_iter(symbol='ETHBTC',
start_str='30 minutes ago UTC')
agg_trade_list = list(agg_trades)
df2 = pd.DataFrame(agg_trade_list)
trade_columns = [
'BestPriceMatch', 'Time', 'Id', 'FirstId', 'LastId', 'BuyerMaker', 'Price',
'Quantity'
]
df2.columns = trade_columns
df2['Time'] = pd.to_datetime(df2['Time'], unit='ms')
df2.to_csv('trades_data.csv', index=False)
# Get market depth(aka orderbook)
depth = client.get_order_book(symbol='BNBBTC')
df3 = pd.DataFrame(depth)
df3[['bids_PRICE', 'bids_QTY']] = pd.DataFrame(df3.bids.tolist(),
index=df3.index)
logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
config_path = '../config/config_testing.ini'
if __name__ == '__main__':
config = configparser.ConfigParser()
config.read(config_path)
api = config['binance']['api']
secret = config['binance']['secret']
binance_client = BinanceClient(api, secret)
historical_trades = binance_client.aggregate_trade_iter(
symbol='XLMBTC', start_str='15 minutes ago UTC')
trade_info = {}
for trade in historical_trades:
first_trade_id_agg = trade['a']
first_trade_id_first = trade['f']
first_trade_id_last = trade['l']
break
pprint(trade)
## WORKS ##
print('Aggregate Trade ID Test:')
try:
historical_trades = binance_client.aggregate_trade_iter(
symbol='XLMBTC', last_id=first_trade_id_agg)
# wait until coin input
tradingPair = input("Coin pair: ").upper() + coinPair
# get trading pair price
price = float(client.get_avg_price(symbol=tradingPair)['price'])
# calculate amount of coin to buy
amountOfCoin = BTCtoSell / price
# rounding the coin to the specified lot size
info = client.get_symbol_info(tradingPair)
minQty = float(info['filters'][2]['minQty'])
amountOfCoin = float_to_string(amountOfCoin, int(-math.log10(minQty)))
# ensure buy limit is setup correctly
# find average price in last 30 mins
agg_trades = client.aggregate_trade_iter(symbol=tradingPair,
start_str='30 minutes ago UTC')
agg_trade_list = list(agg_trades)
total = 0
for trade in agg_trade_list:
fvalue = float(trade['p'])
total = total + fvalue
averagePrice = total / len(agg_trade_list)
minPrice = minQty = float(info['filters'][0]['minPrice'])
averagePrice = float(averagePrice) * buyLimit
averagePrice = float_to_string(averagePrice, int(-math.log10(minPrice)))
# buy order
order = client.order_limit_buy(symbol=tradingPair,
quantity=amountOfCoin,
price=averagePrice)
print('Buy order has been made!')
class TradingBot:
def __init__(self, key: str, secret: str):
self.client = Client(key, secret)
self.algorithm: Union[BasicAlgorithm, None] = None
self.data = pd.DataFrame()
self.request = dict()
self.order = None
# some metadata
self.meta = OrderMetadata()
self.stake_amount = None
self.price = None
self.is_trading = False
self.lot_precision = None
self.price_precision = None
self.roi = dict()
self.stoploss = None
# some helpers
self.data_handler: Union[DataHandler, None] = None
self.logger = None
def create_request(self, pair: str) -> NoReturn:
self.request['symbol'] = pair
self.request['type'] = self.algorithm.order_type
self.request['newOrderRespType'] = 'FULL'
self.request['recvWindow'] = 1000 # why not?
if self.algorithm.order_type == 'LIMIT': # for LIMIT type
self.request['timeInForce'] = 'GTC'
def set_metadata(self, pair: str, stake_amount: int,
algorithm: BasicAlgorithm):
def get_precision(string: str) -> int:
precision = 0
for char in string:
if char == '1':
break
precision += 1 if char == '0' else 0
return precision
self.algorithm = algorithm
self.stake_amount = stake_amount
info = self.client.get_symbol_info(pair)
step_size = info['filters'][2]['stepSize']
self.lot_precision = get_precision(step_size)
tick_size = info['filters'][0]['tickSize']
self.price_precision = get_precision(tick_size)
self.roi = {
float(key): value
for key, value in self.algorithm.roi.items()
}
self.roi[np.inf] = 0
self.stoploss = self.algorithm.stoploss
self.create_request(pair)
self.meta.set_bnb_price(self.client)
self.meta.set_algorithm_name(self.algorithm)
def data_drop(self, state) -> NoReturn:
self.data = self.data.drop(self.data.loc[
self.data["timestamp"] <= time2stamp(state.index[-1])].index)
def roi_stoploss_check(self) -> bool:
diff = time.perf_counter() - self.meta.start_time
keys = np.array(list(self.roi.keys()))
key = np.min(keys[keys * 60 - diff > 0])
profit_price = self.meta.start_price * (1 + self.roi[key])
if self.price > profit_price:
self.meta.set_sell_reason('ROI')
self.logger.info('ROI WAS REACHED')
return True
if self.price < self.meta.start_price * (1 + self.stoploss):
self.meta.set_sell_reason('STOPLOSS')
self.logger.info('STOPLOSS WAS REACHED')
return True
return False
@staticmethod
def process_message(message) -> dict:
message["price"] = message.pop("p")
message["id"] = message.pop("a")
message["amount"] = message.pop("q")
message["timestamp"] = message.pop("T")
message["price"] = pd.to_numeric(message["price"])
message["amount"] = pd.to_numeric(message["amount"])
message["cost"] = message["price"] * message["amount"]
message.pop("E", None)
message.pop("e", None)
message.pop("s", None)
del message["f"]
del message["l"]
del message["m"]
del message["M"]
return message
def update(self, message, act: bool) -> NoReturn:
self.data = self.data.append(message, ignore_index=True)
state = getattr(self.data.bars,
self.algorithm.tick_type)(self.algorithm.tick_size)
if not state.empty:
self.algorithm.set_state(state)
self.logger.info('STATE IS UPDATED')
action = self.algorithm.action(self.is_trading)
if act and action:
if action == 'SELL':
self.meta.set_sell_reason('SELL SIGNAL')
self.act(action, self.algorithm.order_type)
self.data_drop(state)
def get_historical_data(self, pair: str, days: int,
override: bool) -> NoReturn:
path = os.path.abspath(__file__)
path = "/".join(path.split('/')[:-2]) + '/historical_data/'
self.data = fm.load_dataset(client=self.client,
pair=pair,
days=days,
path=path,
override=override)
state = getattr(self.data.bars,
self.algorithm.tick_type)(self.algorithm.tick_size)
self.algorithm.set_state(state)
last_id = self.data.iloc[-1, 0]
self.data_drop(state)
# cycle below is just to minimize lag that occurs because of loading dataset
for i in range(5):
if not self.data.empty:
last_id = self.data.iloc[-1, 0]
agg_trades = self.client.aggregate_trade_iter(symbol=pair,
last_id=last_id)
agg_trades = list(agg_trades)
messages = [
self.process_message(message) for message in agg_trades
]
self.update(messages, False)
def handle_order(self, message) -> NoReturn:
if message['e'] == 'executionReport':
self.meta.add_socket_order(message)
if message['S'] == 'BUY':
self.is_trading = True
elif message['S'] == 'SELL' and message['X'] == 'FILLED':
self.data_handler.update(vars(self.meta))
self.logger.info('DATABASE WAS UPDATED')
self.logger.debug(vars(self.meta))
self.meta.flush()
self.meta.set_bnb_price(self.client)
self.is_trading = False
elif message['e'] == 'error':
self.logger.error('HANDLE ORDER ERROR ' + message['m'])
sys.exit()
def handle_message(self, message) -> NoReturn:
message = self.process_message(message)
self.price = float(message["price"])
# handling roi or stoploss case
if self.is_trading:
if self.roi_stoploss_check():
self.act('SELL', 'MARKET')
self.update(message, True)
def make_limit_request(self, action: str) -> NoReturn:
self.request['side'] = action
# may be better to use self.price + n * tick_size instead of self.algorithm.price
# or IOC with self.price - n * tick_size
self.request['price'] = "{:0.0{}f}".format(self.algorithm.price,
self.price_precision)
if action == 'BUY':
self.meta.set_start_price(float(self.request['price']))
self.meta.set_quantity("{:0.0{}f}".format(
self.stake_amount / self.algorithm.price, self.lot_precision))
self.request['quantity'] = self.meta.get_quantity()
else:
self.meta.set_end_price(float(self.request['price']))
def make_market_request(self, action: str) -> NoReturn:
self.request['side'] = action
if action == 'BUY':
self.meta.set_quantity("{:0.0{}f}".format(
self.stake_amount / self.price, self.lot_precision))
self.request['quantity'] = self.meta.get_quantity()
def act(self, action: str, type_order: str) -> NoReturn:
assert type_order in ['MARKET', 'LIMIT']
assert action in ['BUY', 'SELL']
if type_order == 'MARKET':
self.make_market_request(action)
else:
self.make_limit_request(action)
try:
self.logger.info(action + ' ' + type_order + ' ORDER WAS SENT')
self.logger.debug(self.request)
self.order = self.client.create_order(**self.request)
self.logger.debug(self.order)
except BinanceAPIException as e:
self.logger.error(e)
sys.exit()
self.meta.add_order(self.order)
def trade(self,
pair: str,
days: int,
algorithm: BasicAlgorithm,
override: bool = True,
stake_amount: int = 10) -> NoReturn:
"""
This function is used for trading
:param pair: pair to trade on
:param days: number of days from which data is collected
:param algorithm: algorithm to trade on
:param override: should data be override ?
:param stake_amount: amount of one stake
:return: trade function has no return but it saves logs
"""
self.logger = get_logger('trade')
self.set_metadata(pair, stake_amount, algorithm)
self.data_handler = DataHandler('trade')
self.get_historical_data(pair, days, override)
self.logger.info(pair + ' historical data for ' + str(days) +
' days was downloaded and processed')
bm_user = BinanceSocketManager(self.client)
bm_user.start_user_socket(self.handle_order)
bm_user.start()
bm_trades = BinanceSocketManager(self.client)
conn_key = bm_trades.start_aggtrade_socket(pair, self.handle_message)
bm_trades.start()
tickers = client.get_ticker()
print(tickers[0])
print([t["symbol"] for t in tickers])
depth = client.get_order_book(symbol=SYMBOL, limit=10)
print(depth["bids"], depth["asks"]) # 100단위 호가를 줌..!
trades = client.get_recent_trades(symbol=SYMBOL)
# pprint(len(trades)) # 최근 500틱
h_trades = client.get_historical_trades(symbol=SYMBOL)
# pprint(len(h_trades)) # 최근 500틱? 위 함수랑 무슨 차이지
# Aggregate Trade Iterator ( Backtesting 할때 써먹으면 딱일듯! Generator!)
agg_trades = client.aggregate_trade_iter(symbol=SYMBOL,
start_str='1 years ago UTC')
# for i, trade in enumerate(agg_trades):
# print(i, trade)
# Kline Generator ( Backtesting ㄱㄱ)
# for kline in client.get_historical_klines_generator("BNBBTC", Client.KLINE_INTERVAL_1MINUTE, "1 day ago UTC")
# print(kline)
# do something with the kline
# Kline Historical
# fetch 30 minute klines for the last month of 2017
klines = client.get_historical_klines(symbol=SYMBOL,
interval=Client.KLINE_INTERVAL_30MINUTE)
print(klines[0])
# SYMBOL, Client.KLINE_INTERVAL_30MINUTE, "1 Dec, 2020", "1 Jan, 2021"
class DataEndPoint:
"""Tools for connecting to exchange"""
def __init__(self):
self.client = Client(API_KEY, API_SECRET)
self.db = Db()
def fetch_market_depth(self, symbol='TUSDBTC'):
"""bid and ask prices for given time and symbol"""
depth = self.client.get_order_book(symbol=symbol)
df_bid = pd.DataFrame(depth['bids'],
columns=['price', 'amount',
'col3']).drop(labels=['col3'], axis=1)
df_bid['updatedId'] = depth['lastUpdateId']
df_bid['myUtc'] = dt.datetime.utcnow()
df_bid['symbol'] = symbol
df_bid['price'] = df_bid['price'].astype(float)
df_bid['amount'] = df_bid['amount'].astype(float)
self.db.insert_bid_ask(df_bid, 'bid')
df_ask = pd.DataFrame(depth['asks'],
columns=['price', 'amount',
'col3']).drop(labels=['col3'], axis=1)
df_ask['updatedId'] = depth['lastUpdateId']
df_ask['myUtc'] = dt.datetime.utcnow()
df_ask['symbol'] = symbol
df_ask['price'] = df_ask['price'].astype(float)
df_ask['amount'] = df_ask['amount'].astype(float)
self.db.insert_bid_ask(df_ask, 'ask')
def fetch_all_symbol_prices(self):
"""prices and all symbols in the given time"""
prices = self.client.get_all_tickers()
df = pd.DataFrame(prices)
self.db.write_df(df=df, index=True, table_name='symbols')
def fetch_exchange_info(self):
"""
Exchange info includes
* rateLimits - limits per time intervals - minute, second and day
* symbols - large set limiting factors for symbols - tick sizes, min prices,
orders, base and quote assets, quote precision, status, full symbol
interpretation for a currency pair
* current server time
* time zone
* exchangeFilters - blank field at the moment
"""
info = self.client.get_exchange_info()
# info.keys()
# [u'rateLimits', u'timezone', u'exchangeFilters', u'serverTime', u'symbols']
df = pd.DataFrame(info['rateLimits'])
self.db.write_df(df=df, index=True, table_name='limits')
df = pd.DataFrame(info['symbols'])
df1 = df[['baseAsset', 'filters']]
self.db.write_df(df=df1, index=True, table_name='symbolFilters')
df.drop(labels='filters', inplace=True, axis=1)
self.db.write_df(df=df1, index=True, table_name='symbolInfo')
# unused fields
# info['serverTime']
# info['timezone']
# # UTC
# info['exchangeFilters']
self.client.get_all_orders(symbol='TUSDBTC',
requests_params={'timeout': 5})
def fetch_general_end_points(self):
"""
ping - returns nothing for me
server time only
symbol info - limits for a specific symbol
"""
self.client.ping()
time_res = self.client.get_server_time()
status = self.client.get_system_status()
# this is the same as exchange info
info = self.client.get_symbol_info('TUSDBTC')
pd.DataFrame(info)
def fetch_recent_trades(self):
"""
500 recent trades for the symbol
columns:
id isBestMatch isBuyerMaker price qty time
"""
trades = self.client.get_recent_trades(symbol='TUSDBTC')
pd.DataFrame(trades)
def fetch_historical_trades(self):
"""
seems the same as recent trades
"""
trades = self.client.get_historical_trades(symbol='TUSDBTC')
pd.DataFrame(trades)
def fetch_aggregate_trades(self):
"""not sure what this does, some trade summary but uses letters as column headers"""
trades = self.client.get_aggregate_trades(symbol='TUSDBTC')
pd.DataFrame(trades)
def fetch_aggregate_trade_iterator(self):
agg_trades = self.client.aggregate_trade_iter(
symbol='TUSDBTC', start_str='30 minutes ago UTC')
# iterate over the trade iterator
for trade in agg_trades:
print(trade)
# do something with the trade data
# convert the iterator to a list
# note: generators can only be iterated over once so we need to call it again
agg_trades = self.client.aggregate_trade_iter(
symbol='ETHBTC', start_str='30 minutes ago UTC')
agg_trade_list = list(agg_trades)
# example using last_id value - don't run this one - can be very slow
agg_trades = self.client.aggregate_trade_iter(symbol='ETHBTC',
last_id=3263487)
agg_trade_list = list(agg_trades)
def fetch_candlesticks(self):
candles = self.client.get_klines(
symbol='BNBBTC', interval=Client.KLINE_INTERVAL_30MINUTE)
# this works but I am not sure how to use it
for kline in self.client.get_historical_klines_generator(
"BNBBTC", Client.KLINE_INTERVAL_1MINUTE, "1 day ago UTC"):
print(kline)
# do something with the kline
def fetch_24hr_ticker(self):
"""
summary of price movements for all symbols
"""
tickers = self.get_ticker()
pd.DataFrame(tickers).columns
# [u'askPrice', u'askQty', u'bidPrice', u'bidQty', u'closeTime', u'count',
# u'firstId', u'highPrice', u'lastId', u'lastPrice', u'lastQty',
# u'lowPrice', u'openPrice', u'openTime', u'prevClosePrice',
# u'priceChange', u'priceChangePercent', u'quoteVolume', u'symbol',
# u'volume', u'weightedAvgPrice']
def fetch_orderbook_tickers(client):
"""
summary of current orderbook for all markets
askPrice askQty bidPrice bidQty symbol
"""
tickers = client.get_orderbook_tickers()
pd.DataFrame(tickers)