def populate_historical(market, start_time): binance_api = config['binance']['api'] binance_secret = config['binance']['secret'] binance_client = BinanceClient(binance_api, binance_secret) # Get historical aggregated trade data as generator object and count number of historical trades historical_trades = binance_client.aggregate_trade_iter( symbol=market, start_str=start_time) logger.info('Counting historical trades for database population.') trade_count = sum(1 for trade in historical_trades) logger.debug('trade_count: ' + str(trade_count)) # Get historical aggregated trade data again to refresh generator object (May make total count off by few trades) historical_trades = binance_client.aggregate_trade_iter( symbol=market, start_str=start_time) count = 0 for trade in historical_trades: process_result = process_message(trade, populate=True, symbol=market) if process_result == False: logger.info('Database population complete.') break else: count += 1 logger.info('Processed ' + str(count) + ' of ~' + str(trade_count) + ' historical trades.')
def populate_historical(exchange, market, start_time): if exchange == 'binance': binance_api = config['binance']['api'] binance_secret = config['binance']['secret'] binance_client = BinanceClient(binance_api, binance_secret) # Get historical aggregated trade data as generator object and count number of historical trades historical_trades = binance_client.aggregate_trade_iter( symbol=market, start_str=start_time) logger.info('Counting historical trades for database population.') trade_count = sum(1 for trade in historical_trades) logger.debug('trade_count: ' + str(trade_count)) # Get historical aggregated trade data again to refresh generator object (May make total count off by few trades) historical_trades = binance_client.aggregate_trade_iter( symbol=market, start_str=start_time) count = 0 for trade in historical_trades: process_result = process_message(trade, populate=True, exchange=exchange, market=market) if process_result == False: logger.info('Database population complete.') break #logger.info('Trade document already present in database.') else: count += 1 completion_percentage = "{:.2f}".format( (count / trade_count) * 100) logger.info('Processed ' + str(count) + ' of ~' + str(trade_count) + ' historical trades. [' + completion_percentage + '%]') elif exchange == 'poloniex': logger.warning('POLONIEX DATABASE POPULATION NOT YET IMPLEMENTED.') else: logger.error( 'Unrecognized exchange passed to populate_historical(). Exiting.') sys.exit(1)
sys.exit(1) else: logger.info('Selected Binance market ' + user_market + '.') if backtest_duration == None: backtest_duration = input( 'Input length of time for trade data backtesting/analysis (ex. 30 seconds/9 minutes/3 hours/2 days/1 week): ' ) test_duration = backtest_duration + ' ago UTC' logger.debug('test_duration: ' + test_duration) try: logger.debug( 'Testing user-provided historical data population input.') historical_trades = binance_client.aggregate_trade_iter( symbol=user_market, start_str=test_duration) logger.debug('Attempting count of trades in generator object.') trade_count = sum(1 for trade in historical_trades) populate_duration = test_duration logger.debug('populate_duration: ' + populate_duration) except: logger.error( 'Invalid input for start of historical data population. Exiting.' ) sys.exit(1) if user_market == None or backtest_duration == None: logger.error('Failed to gather valid user input. Exiting.') sys.exit(1) ## Delete existing data for market from database ##
from binance.client import Client from binance.enums import * import matplotlib.pyplot as plt import numpy as np import csv import pandas as pd import seaborn as sns import datetime # getting recent trades client = Client( "wDPuqBa0zeqQRHE26takNQ5G9jLyyFkWxisrheBqmxHDhz4RPcNzR8bPLg4E4Gka", "NwUcys7q7NLPl1ISoIAB97wyAjGNOc2Kw3nJqRJ8BwzFO4ERAsCqdOnK1eDzaPQk") agg_trades = client.aggregate_trade_iter(symbol='ETHBTC', start_str='15 seconds ago UTC') # only extracting the date and price trades = [] for t in agg_trades: t = [t['T'], t['p']] t[0] = datetime.datetime.utcfromtimestamp(round(t[0] / 1000)) split_t = str(t[0]).split(' ') t[0] = split_t[1] trades.append(t) # creating a csv file with the trades data with open('recent_trades.csv', 'w', newline='') as csvfile: filewriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
# Get candle data(aka kline, histo) klines = client.get_historical_klines("BNBBTC", Client.KLINE_INTERVAL_1MINUTE, "1 day ago UTC") df = pd.DataFrame(klines) columns = [ 'OpenTime', 'Open', 'High', 'Low', 'Close', 'Volume', 'CloseTime', 'QuoteVolume', 'NumberTrades', 'BaseVolumne', 'QuoteVolumne', 'Ignored' ] df.columns = columns df['OpenTime'] = pd.to_datetime(df['OpenTime'], unit='ms') df['CloseTime'] = pd.to_datetime(df['CloseTime'], unit='ms') df.to_csv('klines_data.csv', index=False) # Get transactions(aka trades) agg_trades = client.aggregate_trade_iter(symbol='ETHBTC', start_str='30 minutes ago UTC') agg_trade_list = list(agg_trades) df2 = pd.DataFrame(agg_trade_list) trade_columns = [ 'BestPriceMatch', 'Time', 'Id', 'FirstId', 'LastId', 'BuyerMaker', 'Price', 'Quantity' ] df2.columns = trade_columns df2['Time'] = pd.to_datetime(df2['Time'], unit='ms') df2.to_csv('trades_data.csv', index=False) # Get market depth(aka orderbook) depth = client.get_order_book(symbol='BNBBTC') df3 = pd.DataFrame(depth) df3[['bids_PRICE', 'bids_QTY']] = pd.DataFrame(df3.bids.tolist(), index=df3.index)
logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) config_path = '../config/config_testing.ini' if __name__ == '__main__': config = configparser.ConfigParser() config.read(config_path) api = config['binance']['api'] secret = config['binance']['secret'] binance_client = BinanceClient(api, secret) historical_trades = binance_client.aggregate_trade_iter( symbol='XLMBTC', start_str='15 minutes ago UTC') trade_info = {} for trade in historical_trades: first_trade_id_agg = trade['a'] first_trade_id_first = trade['f'] first_trade_id_last = trade['l'] break pprint(trade) ## WORKS ## print('Aggregate Trade ID Test:') try: historical_trades = binance_client.aggregate_trade_iter( symbol='XLMBTC', last_id=first_trade_id_agg)
# wait until coin input tradingPair = input("Coin pair: ").upper() + coinPair # get trading pair price price = float(client.get_avg_price(symbol=tradingPair)['price']) # calculate amount of coin to buy amountOfCoin = BTCtoSell / price # rounding the coin to the specified lot size info = client.get_symbol_info(tradingPair) minQty = float(info['filters'][2]['minQty']) amountOfCoin = float_to_string(amountOfCoin, int(-math.log10(minQty))) # ensure buy limit is setup correctly # find average price in last 30 mins agg_trades = client.aggregate_trade_iter(symbol=tradingPair, start_str='30 minutes ago UTC') agg_trade_list = list(agg_trades) total = 0 for trade in agg_trade_list: fvalue = float(trade['p']) total = total + fvalue averagePrice = total / len(agg_trade_list) minPrice = minQty = float(info['filters'][0]['minPrice']) averagePrice = float(averagePrice) * buyLimit averagePrice = float_to_string(averagePrice, int(-math.log10(minPrice))) # buy order order = client.order_limit_buy(symbol=tradingPair, quantity=amountOfCoin, price=averagePrice) print('Buy order has been made!')
class TradingBot: def __init__(self, key: str, secret: str): self.client = Client(key, secret) self.algorithm: Union[BasicAlgorithm, None] = None self.data = pd.DataFrame() self.request = dict() self.order = None # some metadata self.meta = OrderMetadata() self.stake_amount = None self.price = None self.is_trading = False self.lot_precision = None self.price_precision = None self.roi = dict() self.stoploss = None # some helpers self.data_handler: Union[DataHandler, None] = None self.logger = None def create_request(self, pair: str) -> NoReturn: self.request['symbol'] = pair self.request['type'] = self.algorithm.order_type self.request['newOrderRespType'] = 'FULL' self.request['recvWindow'] = 1000 # why not? if self.algorithm.order_type == 'LIMIT': # for LIMIT type self.request['timeInForce'] = 'GTC' def set_metadata(self, pair: str, stake_amount: int, algorithm: BasicAlgorithm): def get_precision(string: str) -> int: precision = 0 for char in string: if char == '1': break precision += 1 if char == '0' else 0 return precision self.algorithm = algorithm self.stake_amount = stake_amount info = self.client.get_symbol_info(pair) step_size = info['filters'][2]['stepSize'] self.lot_precision = get_precision(step_size) tick_size = info['filters'][0]['tickSize'] self.price_precision = get_precision(tick_size) self.roi = { float(key): value for key, value in self.algorithm.roi.items() } self.roi[np.inf] = 0 self.stoploss = self.algorithm.stoploss self.create_request(pair) self.meta.set_bnb_price(self.client) self.meta.set_algorithm_name(self.algorithm) def data_drop(self, state) -> NoReturn: self.data = self.data.drop(self.data.loc[ self.data["timestamp"] <= time2stamp(state.index[-1])].index) def roi_stoploss_check(self) -> bool: diff = time.perf_counter() - self.meta.start_time keys = np.array(list(self.roi.keys())) key = np.min(keys[keys * 60 - diff > 0]) profit_price = self.meta.start_price * (1 + self.roi[key]) if self.price > profit_price: self.meta.set_sell_reason('ROI') self.logger.info('ROI WAS REACHED') return True if self.price < self.meta.start_price * (1 + self.stoploss): self.meta.set_sell_reason('STOPLOSS') self.logger.info('STOPLOSS WAS REACHED') return True return False @staticmethod def process_message(message) -> dict: message["price"] = message.pop("p") message["id"] = message.pop("a") message["amount"] = message.pop("q") message["timestamp"] = message.pop("T") message["price"] = pd.to_numeric(message["price"]) message["amount"] = pd.to_numeric(message["amount"]) message["cost"] = message["price"] * message["amount"] message.pop("E", None) message.pop("e", None) message.pop("s", None) del message["f"] del message["l"] del message["m"] del message["M"] return message def update(self, message, act: bool) -> NoReturn: self.data = self.data.append(message, ignore_index=True) state = getattr(self.data.bars, self.algorithm.tick_type)(self.algorithm.tick_size) if not state.empty: self.algorithm.set_state(state) self.logger.info('STATE IS UPDATED') action = self.algorithm.action(self.is_trading) if act and action: if action == 'SELL': self.meta.set_sell_reason('SELL SIGNAL') self.act(action, self.algorithm.order_type) self.data_drop(state) def get_historical_data(self, pair: str, days: int, override: bool) -> NoReturn: path = os.path.abspath(__file__) path = "/".join(path.split('/')[:-2]) + '/historical_data/' self.data = fm.load_dataset(client=self.client, pair=pair, days=days, path=path, override=override) state = getattr(self.data.bars, self.algorithm.tick_type)(self.algorithm.tick_size) self.algorithm.set_state(state) last_id = self.data.iloc[-1, 0] self.data_drop(state) # cycle below is just to minimize lag that occurs because of loading dataset for i in range(5): if not self.data.empty: last_id = self.data.iloc[-1, 0] agg_trades = self.client.aggregate_trade_iter(symbol=pair, last_id=last_id) agg_trades = list(agg_trades) messages = [ self.process_message(message) for message in agg_trades ] self.update(messages, False) def handle_order(self, message) -> NoReturn: if message['e'] == 'executionReport': self.meta.add_socket_order(message) if message['S'] == 'BUY': self.is_trading = True elif message['S'] == 'SELL' and message['X'] == 'FILLED': self.data_handler.update(vars(self.meta)) self.logger.info('DATABASE WAS UPDATED') self.logger.debug(vars(self.meta)) self.meta.flush() self.meta.set_bnb_price(self.client) self.is_trading = False elif message['e'] == 'error': self.logger.error('HANDLE ORDER ERROR ' + message['m']) sys.exit() def handle_message(self, message) -> NoReturn: message = self.process_message(message) self.price = float(message["price"]) # handling roi or stoploss case if self.is_trading: if self.roi_stoploss_check(): self.act('SELL', 'MARKET') self.update(message, True) def make_limit_request(self, action: str) -> NoReturn: self.request['side'] = action # may be better to use self.price + n * tick_size instead of self.algorithm.price # or IOC with self.price - n * tick_size self.request['price'] = "{:0.0{}f}".format(self.algorithm.price, self.price_precision) if action == 'BUY': self.meta.set_start_price(float(self.request['price'])) self.meta.set_quantity("{:0.0{}f}".format( self.stake_amount / self.algorithm.price, self.lot_precision)) self.request['quantity'] = self.meta.get_quantity() else: self.meta.set_end_price(float(self.request['price'])) def make_market_request(self, action: str) -> NoReturn: self.request['side'] = action if action == 'BUY': self.meta.set_quantity("{:0.0{}f}".format( self.stake_amount / self.price, self.lot_precision)) self.request['quantity'] = self.meta.get_quantity() def act(self, action: str, type_order: str) -> NoReturn: assert type_order in ['MARKET', 'LIMIT'] assert action in ['BUY', 'SELL'] if type_order == 'MARKET': self.make_market_request(action) else: self.make_limit_request(action) try: self.logger.info(action + ' ' + type_order + ' ORDER WAS SENT') self.logger.debug(self.request) self.order = self.client.create_order(**self.request) self.logger.debug(self.order) except BinanceAPIException as e: self.logger.error(e) sys.exit() self.meta.add_order(self.order) def trade(self, pair: str, days: int, algorithm: BasicAlgorithm, override: bool = True, stake_amount: int = 10) -> NoReturn: """ This function is used for trading :param pair: pair to trade on :param days: number of days from which data is collected :param algorithm: algorithm to trade on :param override: should data be override ? :param stake_amount: amount of one stake :return: trade function has no return but it saves logs """ self.logger = get_logger('trade') self.set_metadata(pair, stake_amount, algorithm) self.data_handler = DataHandler('trade') self.get_historical_data(pair, days, override) self.logger.info(pair + ' historical data for ' + str(days) + ' days was downloaded and processed') bm_user = BinanceSocketManager(self.client) bm_user.start_user_socket(self.handle_order) bm_user.start() bm_trades = BinanceSocketManager(self.client) conn_key = bm_trades.start_aggtrade_socket(pair, self.handle_message) bm_trades.start()
tickers = client.get_ticker() print(tickers[0]) print([t["symbol"] for t in tickers]) depth = client.get_order_book(symbol=SYMBOL, limit=10) print(depth["bids"], depth["asks"]) # 100단위 호가를 줌..! trades = client.get_recent_trades(symbol=SYMBOL) # pprint(len(trades)) # 최근 500틱 h_trades = client.get_historical_trades(symbol=SYMBOL) # pprint(len(h_trades)) # 최근 500틱? 위 함수랑 무슨 차이지 # Aggregate Trade Iterator ( Backtesting 할때 써먹으면 딱일듯! Generator!) agg_trades = client.aggregate_trade_iter(symbol=SYMBOL, start_str='1 years ago UTC') # for i, trade in enumerate(agg_trades): # print(i, trade) # Kline Generator ( Backtesting ㄱㄱ) # for kline in client.get_historical_klines_generator("BNBBTC", Client.KLINE_INTERVAL_1MINUTE, "1 day ago UTC") # print(kline) # do something with the kline # Kline Historical # fetch 30 minute klines for the last month of 2017 klines = client.get_historical_klines(symbol=SYMBOL, interval=Client.KLINE_INTERVAL_30MINUTE) print(klines[0]) # SYMBOL, Client.KLINE_INTERVAL_30MINUTE, "1 Dec, 2020", "1 Jan, 2021"
class DataEndPoint: """Tools for connecting to exchange""" def __init__(self): self.client = Client(API_KEY, API_SECRET) self.db = Db() def fetch_market_depth(self, symbol='TUSDBTC'): """bid and ask prices for given time and symbol""" depth = self.client.get_order_book(symbol=symbol) df_bid = pd.DataFrame(depth['bids'], columns=['price', 'amount', 'col3']).drop(labels=['col3'], axis=1) df_bid['updatedId'] = depth['lastUpdateId'] df_bid['myUtc'] = dt.datetime.utcnow() df_bid['symbol'] = symbol df_bid['price'] = df_bid['price'].astype(float) df_bid['amount'] = df_bid['amount'].astype(float) self.db.insert_bid_ask(df_bid, 'bid') df_ask = pd.DataFrame(depth['asks'], columns=['price', 'amount', 'col3']).drop(labels=['col3'], axis=1) df_ask['updatedId'] = depth['lastUpdateId'] df_ask['myUtc'] = dt.datetime.utcnow() df_ask['symbol'] = symbol df_ask['price'] = df_ask['price'].astype(float) df_ask['amount'] = df_ask['amount'].astype(float) self.db.insert_bid_ask(df_ask, 'ask') def fetch_all_symbol_prices(self): """prices and all symbols in the given time""" prices = self.client.get_all_tickers() df = pd.DataFrame(prices) self.db.write_df(df=df, index=True, table_name='symbols') def fetch_exchange_info(self): """ Exchange info includes * rateLimits - limits per time intervals - minute, second and day * symbols - large set limiting factors for symbols - tick sizes, min prices, orders, base and quote assets, quote precision, status, full symbol interpretation for a currency pair * current server time * time zone * exchangeFilters - blank field at the moment """ info = self.client.get_exchange_info() # info.keys() # [u'rateLimits', u'timezone', u'exchangeFilters', u'serverTime', u'symbols'] df = pd.DataFrame(info['rateLimits']) self.db.write_df(df=df, index=True, table_name='limits') df = pd.DataFrame(info['symbols']) df1 = df[['baseAsset', 'filters']] self.db.write_df(df=df1, index=True, table_name='symbolFilters') df.drop(labels='filters', inplace=True, axis=1) self.db.write_df(df=df1, index=True, table_name='symbolInfo') # unused fields # info['serverTime'] # info['timezone'] # # UTC # info['exchangeFilters'] self.client.get_all_orders(symbol='TUSDBTC', requests_params={'timeout': 5}) def fetch_general_end_points(self): """ ping - returns nothing for me server time only symbol info - limits for a specific symbol """ self.client.ping() time_res = self.client.get_server_time() status = self.client.get_system_status() # this is the same as exchange info info = self.client.get_symbol_info('TUSDBTC') pd.DataFrame(info) def fetch_recent_trades(self): """ 500 recent trades for the symbol columns: id isBestMatch isBuyerMaker price qty time """ trades = self.client.get_recent_trades(symbol='TUSDBTC') pd.DataFrame(trades) def fetch_historical_trades(self): """ seems the same as recent trades """ trades = self.client.get_historical_trades(symbol='TUSDBTC') pd.DataFrame(trades) def fetch_aggregate_trades(self): """not sure what this does, some trade summary but uses letters as column headers""" trades = self.client.get_aggregate_trades(symbol='TUSDBTC') pd.DataFrame(trades) def fetch_aggregate_trade_iterator(self): agg_trades = self.client.aggregate_trade_iter( symbol='TUSDBTC', start_str='30 minutes ago UTC') # iterate over the trade iterator for trade in agg_trades: print(trade) # do something with the trade data # convert the iterator to a list # note: generators can only be iterated over once so we need to call it again agg_trades = self.client.aggregate_trade_iter( symbol='ETHBTC', start_str='30 minutes ago UTC') agg_trade_list = list(agg_trades) # example using last_id value - don't run this one - can be very slow agg_trades = self.client.aggregate_trade_iter(symbol='ETHBTC', last_id=3263487) agg_trade_list = list(agg_trades) def fetch_candlesticks(self): candles = self.client.get_klines( symbol='BNBBTC', interval=Client.KLINE_INTERVAL_30MINUTE) # this works but I am not sure how to use it for kline in self.client.get_historical_klines_generator( "BNBBTC", Client.KLINE_INTERVAL_1MINUTE, "1 day ago UTC"): print(kline) # do something with the kline def fetch_24hr_ticker(self): """ summary of price movements for all symbols """ tickers = self.get_ticker() pd.DataFrame(tickers).columns # [u'askPrice', u'askQty', u'bidPrice', u'bidQty', u'closeTime', u'count', # u'firstId', u'highPrice', u'lastId', u'lastPrice', u'lastQty', # u'lowPrice', u'openPrice', u'openTime', u'prevClosePrice', # u'priceChange', u'priceChangePercent', u'quoteVolume', u'symbol', # u'volume', u'weightedAvgPrice'] def fetch_orderbook_tickers(client): """ summary of current orderbook for all markets askPrice askQty bidPrice bidQty symbol """ tickers = client.get_orderbook_tickers() pd.DataFrame(tickers)