def get_order_book_huobi_url(pair_name): final_url = HUOBI_GET_ORDER_BOOK + pair_name + "&type=step0" if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def check_highest_bid_bigger_than_lowest_ask(first_one, second_one, threshold): if not first_one.bid or not second_one.ask: return difference = get_change(first_one.bid, second_one.ask, provide_abs=False) if should_print_debug(): msg = """check_highest_bid_bigger_than_lowest_ask called for threshold = {threshold} BID: {bid:.7f} AKS: {ask:.7f} DIFF: {diff:.7f} """.format(threshold=threshold, bid=first_one.bid, ask=second_one.ask, diff=difference) print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP) if difference >= threshold: factual_threshold = threshold severity_flag = "" if 5.0 < difference < 10.0: severity_flag = "<b> ! ACT NOW ! </b>" factual_threshold = 5.0 elif difference > 10.0: severity_flag = "<b>!!! ACT IMMEDIATELY !!!</b>" factual_threshold = 10.0 msg = """{severity_flag} highest bid bigger than Lowest ask for more than {num} - <b>{diff:.7f}</b>""".format( severity_flag=severity_flag, num=factual_threshold, diff=difference) return msg, first_one.pair_id, first_one, second_one return ()
def get_ticker_huobi_url(pair_name): final_url = HUOBI_GET_TICKER + pair_name if should_print_debug(): print_to_console(final_url, LOG_ALL_OTHER_STUFF) return final_url
def get_tickers_binance_url(pair_name): final_url = BINANCE_GET_TICKER if should_print_debug(): print_to_console(final_url, LOG_ALL_OTHER_STUFF) return final_url
def get_ticker_kraken_url(pair_name): # https://api.kraken.com/0/public/Ticker?pair=DASHXBT final_url = KRAKEN_GET_TICKER + pair_name if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_order_book_bittrex_url(pair_name): # https://bittrex.com/api/v1.1/public/getorderbook?type=both&market=BTC-LTC final_url = BITTREX_GET_ORDER_BOOK + pair_name if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_order_book_poloniex_url(pair_name): # https://poloniex.com/public?command=returnOrderBook¤cyPair=BTC_NXT&depth=10, depth optional final_url = POLONIEX_GET_ORDER_BOOK + pair_name if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_history_bittrex_url(pair_name, prev_time, now_time): # https://bittrex.com/api/v1.1/public/getmarkethistory?market=BTC-LTC final_url = BITTREX_GET_HISTORY + pair_name + "&since=" + str(prev_time) if should_print_debug(): print_to_console(final_url, LOG_ALL_OTHER_STUFF) return final_url
def get_ticker_bittrex_url(pair_name): # https://bittrex.com/api/v1.1/public/getticker?market=BTC-LTC final_url = BITTREX_GET_TICKER + pair_name if should_print_debug(): print_to_console(final_url, LOG_ALL_OTHER_STUFF) return final_url
def get_order_book_kraken_url(pair_name): # https://api.kraken.com/0/public/Depth?pair=XETHXXBT final_url = KRAKEN_GET_ORDER_BOOK + pair_name if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_history_huobi_url(pair_name, date_start, date_end): final_url = HUOBI_GET_HISTORY + pair_name + "&size=1000" if should_print_debug(): print_to_console(final_url, LOG_ALL_OTHER_STUFF) return final_url
def get_order_book_binance_url(currency): # https://api.binance.com/api/v1/depth?symbol=XMRETH final_url = BINANCE_GET_ORDER_BOOK + currency if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_ohlc_poloniex_url(currency, date_start, date_end, period): final_url = POLONIEX_GET_OHLC + currency + "&start=" + str(date_start) + \ "&end=" + str(date_end) + "&period=" + str(period) if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_history_poloniex_url(pair_name, prev_time, now_time): # https://poloniex.com/public?command=returnTradeHistory¤cyPair=BTC_NXT&start=1501693512&end=1501693572 final_url = POLONIEX_GET_HISTORY + pair_name + "&start=" + str( prev_time) + "&end=" + str(now_time) if should_print_debug(): print_to_console(final_url, LOG_ALL_OTHER_STUFF) return final_url
def get_ohlc_huobi_url(pair_name, date_start, date_end, period): date_start_ms = 1000 * date_start # market/history/kline?period=1day&size=200&symbol=btcusdt final_url = HUOBI_GET_OHLC + pair_name + "&period=" + period if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_history_binance_url(pair_name, date_start, date_end): # https://api.binance.com/api/v1/aggTrades?symbol=XMRETH # Optional startTime, endTime final_url = BINANCE_GET_HISTORY + pair_name if should_print_debug(): print_to_console(final_url, LOG_ALL_OTHER_STUFF) return final_url
def get_ohlc_kraken_url(currency, date_start, date_end, period): # https://api.kraken.com/0/public/OHLC?pair=XXRPXXBT&since=1501520850&interval=15 final_url = KRAKEN_GET_OHLC + currency + "&since=" + str( date_start) + "&interval=" + str(period) if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_ohlc_bittrex_url(pair_name, date_start, date_end, period): result_set = [] # https://bittrex.com/Api/v2.0/pub/market/GetTicks?tickInterval=oneMin&marketName=BTC-ETH&_=timest final_url = BITTREX_GET_OHLC + period + "&marketName=" + pair_name + "&_=" + str(date_start) if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_ohlc_binance_url(currency, date_start, date_end, period): date_start_ms = 1000 * date_start # https://api.binance.com/api/v1/klines?symbol=XMRETH&interval=15m&startTime= final_url = BINANCE_GET_OHLC + currency + "&interval=" + period + "&startTime=" + str( date_start_ms) if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_balance_binance_post_details(key): final_url = BINANCE_CHECK_BALANCE body = {"timestamp": get_now_seconds_utc_ms(), "recvWindow": 5000} res = generate_post_request(final_url, body, key) if should_print_debug(): print_to_console(res, LOG_ALL_MARKET_NETWORK_RELATED_CRAP) return res
def get_ticker_poloniex_url(currency_names): """ :param currency_names: for backwards compatibility :param timest: for backwards compatibility :return: """ final_url = POLONIEX_GET_TICKER if should_print_debug(): print_to_console(final_url, LOG_ALL_DEBUG) return final_url
def get_balance_huobi_post_details(key): path = HUOBI_CHECK_BALANCE + get_huobi_account(key) + "/balance" final_url = HUOBI_API_URL + path + "?" body, url = generate_body_and_url_get_request(key, HUOBI_API_ONLY, path) final_url += url res = PostRequestDetails(final_url, HUOBI_GET_HEADERS, body) if should_print_debug(): print_to_console(res, LOG_ALL_MARKET_NETWORK_RELATED_CRAP) return res
def get_diff_lowest_ask_vs_highest_bid(first_one, second_one, threshold): difference = get_change(first_one.ask, second_one.bid) if should_print_debug(): msg = "get_diff_lowest_ask_vs_highest_bid: ASK = {ask} BID = {bid} DIFF={diff}".format( ask=first_one.ask, bid=second_one.bid, diff=difference) print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP) if difference >= threshold: msg = "Lowest ask differ from highest bid more than {num} %".format(num=threshold) return msg, first_one.pair_id, first_one, second_one return ()
def get_balance_poloniex_post_details(key): body = {'command': 'returnCompleteBalances', 'nonce': generate_nonce()} headers = {"Key": key.api_key, "Sign": signed_body(body, key.secret)} # https://poloniex.com/tradingApi final_url = POLONIEX_CHECK_BALANCE res = PostRequestDetails(final_url, headers, body) if should_print_debug(): print_to_console(res, LOG_ALL_MARKET_NETWORK_RELATED_CRAP) return res
def load_all_public_data(args): """ 06.08.2019 As far as I remember it is NOT main data retrieval routine Retrieve ticker, trade history, candles and order book from ALL supported exchanges and store it within DB every TIMEOUT seconds through REST api. Majority of exchanges tend to throttle clients who send too many requests from the same ip - so be mindful about timeout. :param args: :return: """ pg_conn, settings = process_args(args) processor = ConnectionPool() def split_on_errors(raw_response): valid_objects = filter(lambda x: type(x) != str, raw_response) error_strings = filter(lambda x: type(x) != str, raw_response) return valid_objects, error_strings while True: end_time = get_now_seconds_utc() start_time = end_time - POLL_PERIOD_SECONDS candles, errs = split_on_errors(get_ohlc_speedup(start_time, end_time, processor)) bulk_insert_to_postgres(pg_conn, Candle.table_name, Candle.columns, candles) trade_history, errs = split_on_errors(get_history_speedup(start_time, end_time, processor)) bulk_insert_to_postgres(pg_conn, TradeHistory.table_name, TradeHistory.columns, trade_history) tickers, errs = split_on_errors(get_ticker_speedup(end_time, processor)) bulk_insert_to_postgres(pg_conn, Ticker.table_name, Ticker.columns, tickers) if should_print_debug(): msg = """History retrieval at {ts}: Candle size - {num} Ticker size - {num3} Trade history size - {num2} """.format(ts=end_time, num=len(candles), num3=len(tickers), num2=len(trade_history)) print_to_console(msg, LOG_ALL_ERRORS) log_to_file(msg, "candles_trade_history.log") print_to_console("Before sleep...", LOG_ALL_ERRORS) sleep_for(POLL_PERIOD_SECONDS)
def get_history_kraken_url(pair_name, prev_time, now_time): """ :param pair_name: :param prev_time: :param now_time: for backwards compatibility :return: """ # https://api.kraken.com/0/public/Trades?pair=XETHXXBT&since=1501693512 final_url = KRAKEN_GET_HISTORY + pair_name + "&since=" + str(prev_time) if should_print_debug(): print_to_console(final_url, LOG_ALL_OTHER_STUFF) return final_url
def get_balance_bittrex_post_details(key): final_url = BITTREX_CHECK_BALANCE + key.api_key + "&nonce=" + str( generate_nonce()) body = {} final_url += _urlencode(body) headers = {"apisign": signed_string(final_url, key.secret)} res = PostRequestDetails(final_url, headers, body) if should_print_debug(): print_to_console(res, LOG_ALL_MARKET_NETWORK_RELATED_CRAP) return res
def load_order_books(args): """ Periodically retrieve FULL order books from ALL supported exchanges via REST api and save it for further analysis in DB. Under the hood requests are sent in async fashion via gevent library :param args: config file :return: """ pg_conn, _ = process_args(args) processor = ConnectionPool() while True: ts = get_now_seconds_utc() results = get_order_book_speedup(ts, processor) order_book = filter(lambda x: type(x) != str, results) load_to_postgres(order_book, ORDER_BOOK_TYPE_NAME, pg_conn) order_book_size = len(order_book) order_book_ask_size = 0 order_book_bid_size = 0 for entry in order_book: if entry is not None: order_book_ask_size += len(entry.ask) order_book_bid_size += len(entry.bid) if should_print_debug(): msg = """Orderbook retrieval at {tt}: Order book size - {num1} Order book asks - {num10} Order book bids - {num20}""".format( tt=ts, num1=order_book_size, num10=order_book_ask_size, num20=order_book_bid_size) print_to_console(msg, LOG_ALL_ERRORS) log_to_file(msg, "order_book.log") print_to_console("Before sleep...", LOG_ALL_ERRORS) sleep_for(ORDER_BOOK_POLL_TIMEOUT)
def search_for_arbitrage(sell_order_book, buy_order_book, threshold, balance_threshold, action_to_perform, balance_state, deal_cap, type_of_deal, worker_pool, msg_queue): """ :param sell_order_book: order_book from exchange where we are going to SELL :param buy_order_book: order_book from exchange where we are going to BUY :param threshold: difference in price in percent that MAY trigger MUTUAL deal placement :param balance_threshold: for interface compatibility with balance_adjustment method :param action_to_perform: method that will be called in case threshold condition are met :param balance_state: balance across all active exchange for all supported currencies :param deal_cap: dynamically updated minimum volume per currency :param type_of_deal: ARBITRAGE or REVERSE. EXPIRED or FAILED will not be processed here :param worker_pool: gevent based connection pool for speedy deal placement :param msg_queue: redis backed msq queue with notification for Telegram :return: """ deal_status = STATUS.FAILURE, None if not sell_order_book.bid or not buy_order_book.ask: return deal_status difference = get_change(sell_order_book.bid[FIRST].price, buy_order_book.ask[LAST].price, provide_abs=False) if should_print_debug(): log_arbitrage_heart_beat(sell_order_book, buy_order_book, difference) if difference >= threshold: min_volume = determine_minimum_volume(sell_order_book, buy_order_book, balance_state) min_volume = adjust_minimum_volume_by_trading_cap(deal_cap, min_volume) min_volume = adjust_maximum_volume_by_trading_cap(deal_cap, min_volume) min_volume = round_volume_by_exchange_rules( sell_order_book.exchange_id, buy_order_book.exchange_id, min_volume, sell_order_book.pair_id) if min_volume <= 0: log_arbitrage_determined_volume_not_enough(sell_order_book, buy_order_book, msg_queue) return deal_status sell_price = adjust_price_by_order_book(sell_order_book.bid, min_volume) arbitrage_id = get_next_arbitrage_id() create_time = get_now_seconds_utc() trade_at_first_exchange = Trade(DEAL_TYPE.SELL, sell_order_book.exchange_id, sell_order_book.pair_id, sell_price, min_volume, sell_order_book.timest, create_time, arbitrage_id=arbitrage_id) buy_price = adjust_price_by_order_book(buy_order_book.ask, min_volume) trade_at_second_exchange = Trade(DEAL_TYPE.BUY, buy_order_book.exchange_id, buy_order_book.pair_id, buy_price, min_volume, buy_order_book.timest, create_time, arbitrage_id=arbitrage_id) final_difference = get_change(sell_price, buy_price, provide_abs=False) if final_difference <= 0.2: log_arbitrage_determined_price_not_enough( sell_price, sell_order_book.bid[FIRST].price, buy_price, buy_order_book.ask[LAST].price, difference, final_difference, sell_order_book.pair_id, msg_queue) return deal_status trade_pair = TradePair(trade_at_first_exchange, trade_at_second_exchange, sell_order_book.timest, buy_order_book.timest, type_of_deal) placement_status = action_to_perform(trade_pair, final_difference, "history_trades.log", worker_pool, msg_queue) # NOTE: if we can't update balance for more than TIMEOUT seconds arbitrage process will exit # for exchange_id in [trade_pair.deal_1.exchange_id, trade_pair.deal_2.exchange_id]: # update_balance_by_exchange(exchange_id) # deal_status = placement_status, trade_pair return deal_status