def get_order_book_huobi_url(pair_name):
final_url = HUOBI_GET_ORDER_BOOK + pair_name + "&type=step0"
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def check_highest_bid_bigger_than_lowest_ask(first_one, second_one, threshold):
if not first_one.bid or not second_one.ask:
return
difference = get_change(first_one.bid, second_one.ask, provide_abs=False)
if should_print_debug():
msg = """check_highest_bid_bigger_than_lowest_ask called for
threshold = {threshold}
BID: {bid:.7f}
AKS: {ask:.7f}
DIFF: {diff:.7f}
""".format(threshold=threshold, bid=first_one.bid, ask=second_one.ask, diff=difference)
print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP)
if difference >= threshold:
factual_threshold = threshold
severity_flag = ""
if 5.0 < difference < 10.0:
severity_flag = "<b> ! ACT NOW ! </b>"
factual_threshold = 5.0
elif difference > 10.0:
severity_flag = "<b>!!! ACT IMMEDIATELY !!!</b>"
factual_threshold = 10.0
msg = """{severity_flag}
highest bid bigger than Lowest ask for more than {num} - <b>{diff:.7f}</b>""".format(
severity_flag=severity_flag, num=factual_threshold, diff=difference)
return msg, first_one.pair_id, first_one, second_one
return ()
def get_ticker_huobi_url(pair_name):
final_url = HUOBI_GET_TICKER + pair_name
if should_print_debug():
print_to_console(final_url, LOG_ALL_OTHER_STUFF)
return final_url
def get_tickers_binance_url(pair_name):
final_url = BINANCE_GET_TICKER
if should_print_debug():
print_to_console(final_url, LOG_ALL_OTHER_STUFF)
return final_url
def get_ticker_kraken_url(pair_name):
# https://api.kraken.com/0/public/Ticker?pair=DASHXBT
final_url = KRAKEN_GET_TICKER + pair_name
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_order_book_bittrex_url(pair_name):
# https://bittrex.com/api/v1.1/public/getorderbook?type=both&market=BTC-LTC
final_url = BITTREX_GET_ORDER_BOOK + pair_name
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_order_book_poloniex_url(pair_name):
# https://poloniex.com/public?command=returnOrderBook¤cyPair=BTC_NXT&depth=10, depth optional
final_url = POLONIEX_GET_ORDER_BOOK + pair_name
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_history_bittrex_url(pair_name, prev_time, now_time):
# https://bittrex.com/api/v1.1/public/getmarkethistory?market=BTC-LTC
final_url = BITTREX_GET_HISTORY + pair_name + "&since=" + str(prev_time)
if should_print_debug():
print_to_console(final_url, LOG_ALL_OTHER_STUFF)
return final_url
def get_ticker_bittrex_url(pair_name):
# https://bittrex.com/api/v1.1/public/getticker?market=BTC-LTC
final_url = BITTREX_GET_TICKER + pair_name
if should_print_debug():
print_to_console(final_url, LOG_ALL_OTHER_STUFF)
return final_url
def get_order_book_kraken_url(pair_name):
# https://api.kraken.com/0/public/Depth?pair=XETHXXBT
final_url = KRAKEN_GET_ORDER_BOOK + pair_name
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_history_huobi_url(pair_name, date_start, date_end):
final_url = HUOBI_GET_HISTORY + pair_name + "&size=1000"
if should_print_debug():
print_to_console(final_url, LOG_ALL_OTHER_STUFF)
return final_url
def get_order_book_binance_url(currency):
# https://api.binance.com/api/v1/depth?symbol=XMRETH
final_url = BINANCE_GET_ORDER_BOOK + currency
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_ohlc_poloniex_url(currency, date_start, date_end, period):
final_url = POLONIEX_GET_OHLC + currency + "&start=" + str(date_start) + \
"&end=" + str(date_end) + "&period=" + str(period)
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_history_poloniex_url(pair_name, prev_time, now_time):
# https://poloniex.com/public?command=returnTradeHistory¤cyPair=BTC_NXT&start=1501693512&end=1501693572
final_url = POLONIEX_GET_HISTORY + pair_name + "&start=" + str(
prev_time) + "&end=" + str(now_time)
if should_print_debug():
print_to_console(final_url, LOG_ALL_OTHER_STUFF)
return final_url
def get_ohlc_huobi_url(pair_name, date_start, date_end, period):
date_start_ms = 1000 * date_start
# market/history/kline?period=1day&size=200&symbol=btcusdt
final_url = HUOBI_GET_OHLC + pair_name + "&period=" + period
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_history_binance_url(pair_name, date_start, date_end):
# https://api.binance.com/api/v1/aggTrades?symbol=XMRETH
# Optional startTime, endTime
final_url = BINANCE_GET_HISTORY + pair_name
if should_print_debug():
print_to_console(final_url, LOG_ALL_OTHER_STUFF)
return final_url
def get_ohlc_kraken_url(currency, date_start, date_end, period):
# https://api.kraken.com/0/public/OHLC?pair=XXRPXXBT&since=1501520850&interval=15
final_url = KRAKEN_GET_OHLC + currency + "&since=" + str(
date_start) + "&interval=" + str(period)
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_ohlc_bittrex_url(pair_name, date_start, date_end, period):
result_set = []
# https://bittrex.com/Api/v2.0/pub/market/GetTicks?tickInterval=oneMin&marketName=BTC-ETH&_=timest
final_url = BITTREX_GET_OHLC + period + "&marketName=" + pair_name + "&_=" + str(date_start)
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_ohlc_binance_url(currency, date_start, date_end, period):
date_start_ms = 1000 * date_start
# https://api.binance.com/api/v1/klines?symbol=XMRETH&interval=15m&startTime=
final_url = BINANCE_GET_OHLC + currency + "&interval=" + period + "&startTime=" + str(
date_start_ms)
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_balance_binance_post_details(key):
final_url = BINANCE_CHECK_BALANCE
body = {"timestamp": get_now_seconds_utc_ms(), "recvWindow": 5000}
res = generate_post_request(final_url, body, key)
if should_print_debug():
print_to_console(res, LOG_ALL_MARKET_NETWORK_RELATED_CRAP)
return res
def get_ticker_poloniex_url(currency_names):
"""
:param currency_names: for backwards compatibility
:param timest: for backwards compatibility
:return:
"""
final_url = POLONIEX_GET_TICKER
if should_print_debug():
print_to_console(final_url, LOG_ALL_DEBUG)
return final_url
def get_balance_huobi_post_details(key):
path = HUOBI_CHECK_BALANCE + get_huobi_account(key) + "/balance"
final_url = HUOBI_API_URL + path + "?"
body, url = generate_body_and_url_get_request(key, HUOBI_API_ONLY, path)
final_url += url
res = PostRequestDetails(final_url, HUOBI_GET_HEADERS, body)
if should_print_debug():
print_to_console(res, LOG_ALL_MARKET_NETWORK_RELATED_CRAP)
return res
def get_diff_lowest_ask_vs_highest_bid(first_one, second_one, threshold):
difference = get_change(first_one.ask, second_one.bid)
if should_print_debug():
msg = "get_diff_lowest_ask_vs_highest_bid: ASK = {ask} BID = {bid} DIFF={diff}".format(
ask=first_one.ask, bid=second_one.bid, diff=difference)
print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP)
if difference >= threshold:
msg = "Lowest ask differ from highest bid more than {num} %".format(num=threshold)
return msg, first_one.pair_id, first_one, second_one
return ()
def get_balance_poloniex_post_details(key):
body = {'command': 'returnCompleteBalances', 'nonce': generate_nonce()}
headers = {"Key": key.api_key, "Sign": signed_body(body, key.secret)}
# https://poloniex.com/tradingApi
final_url = POLONIEX_CHECK_BALANCE
res = PostRequestDetails(final_url, headers, body)
if should_print_debug():
print_to_console(res, LOG_ALL_MARKET_NETWORK_RELATED_CRAP)
return res
def load_all_public_data(args):
"""
06.08.2019 As far as I remember it is NOT main data retrieval routine
Retrieve ticker, trade history, candles and order book
from ALL supported exchanges
and store it within DB
every TIMEOUT seconds through REST api.
Majority of exchanges tend to throttle clients who send too many requests
from the same ip - so be mindful about timeout.
:param args:
:return:
"""
pg_conn, settings = process_args(args)
processor = ConnectionPool()
def split_on_errors(raw_response):
valid_objects = filter(lambda x: type(x) != str, raw_response)
error_strings = filter(lambda x: type(x) != str, raw_response)
return valid_objects, error_strings
while True:
end_time = get_now_seconds_utc()
start_time = end_time - POLL_PERIOD_SECONDS
candles, errs = split_on_errors(get_ohlc_speedup(start_time, end_time, processor))
bulk_insert_to_postgres(pg_conn, Candle.table_name, Candle.columns, candles)
trade_history, errs = split_on_errors(get_history_speedup(start_time, end_time, processor))
bulk_insert_to_postgres(pg_conn, TradeHistory.table_name, TradeHistory.columns, trade_history)
tickers, errs = split_on_errors(get_ticker_speedup(end_time, processor))
bulk_insert_to_postgres(pg_conn, Ticker.table_name, Ticker.columns, tickers)
if should_print_debug():
msg = """History retrieval at {ts}:
Candle size - {num}
Ticker size - {num3}
Trade history size - {num2}
""".format(ts=end_time, num=len(candles), num3=len(tickers), num2=len(trade_history))
print_to_console(msg, LOG_ALL_ERRORS)
log_to_file(msg, "candles_trade_history.log")
print_to_console("Before sleep...", LOG_ALL_ERRORS)
sleep_for(POLL_PERIOD_SECONDS)
def get_history_kraken_url(pair_name, prev_time, now_time):
"""
:param pair_name:
:param prev_time:
:param now_time: for backwards compatibility
:return:
"""
# https://api.kraken.com/0/public/Trades?pair=XETHXXBT&since=1501693512
final_url = KRAKEN_GET_HISTORY + pair_name + "&since=" + str(prev_time)
if should_print_debug():
print_to_console(final_url, LOG_ALL_OTHER_STUFF)
return final_url
def get_balance_bittrex_post_details(key):
final_url = BITTREX_CHECK_BALANCE + key.api_key + "&nonce=" + str(
generate_nonce())
body = {}
final_url += _urlencode(body)
headers = {"apisign": signed_string(final_url, key.secret)}
res = PostRequestDetails(final_url, headers, body)
if should_print_debug():
print_to_console(res, LOG_ALL_MARKET_NETWORK_RELATED_CRAP)
return res
def load_order_books(args):
"""
Periodically retrieve FULL order books
from ALL supported exchanges via REST api
and save it for further analysis in DB.
Under the hood requests are sent in async fashion via gevent library
:param args: config file
:return:
"""
pg_conn, _ = process_args(args)
processor = ConnectionPool()
while True:
ts = get_now_seconds_utc()
results = get_order_book_speedup(ts, processor)
order_book = filter(lambda x: type(x) != str, results)
load_to_postgres(order_book, ORDER_BOOK_TYPE_NAME, pg_conn)
order_book_size = len(order_book)
order_book_ask_size = 0
order_book_bid_size = 0
for entry in order_book:
if entry is not None:
order_book_ask_size += len(entry.ask)
order_book_bid_size += len(entry.bid)
if should_print_debug():
msg = """Orderbook retrieval at {tt}:
Order book size - {num1} Order book asks - {num10} Order book bids - {num20}""".format(
tt=ts,
num1=order_book_size,
num10=order_book_ask_size,
num20=order_book_bid_size)
print_to_console(msg, LOG_ALL_ERRORS)
log_to_file(msg, "order_book.log")
print_to_console("Before sleep...", LOG_ALL_ERRORS)
sleep_for(ORDER_BOOK_POLL_TIMEOUT)
def search_for_arbitrage(sell_order_book, buy_order_book, threshold,
balance_threshold, action_to_perform, balance_state,
deal_cap, type_of_deal, worker_pool, msg_queue):
"""
:param sell_order_book: order_book from exchange where we are going to SELL
:param buy_order_book: order_book from exchange where we are going to BUY
:param threshold: difference in price in percent that MAY trigger MUTUAL deal placement
:param balance_threshold: for interface compatibility with balance_adjustment method
:param action_to_perform: method that will be called in case threshold condition are met
:param balance_state: balance across all active exchange for all supported currencies
:param deal_cap: dynamically updated minimum volume per currency
:param type_of_deal: ARBITRAGE or REVERSE. EXPIRED or FAILED will not be processed here
:param worker_pool: gevent based connection pool for speedy deal placement
:param msg_queue: redis backed msq queue with notification for Telegram
:return:
"""
deal_status = STATUS.FAILURE, None
if not sell_order_book.bid or not buy_order_book.ask:
return deal_status
difference = get_change(sell_order_book.bid[FIRST].price,
buy_order_book.ask[LAST].price,
provide_abs=False)
if should_print_debug():
log_arbitrage_heart_beat(sell_order_book, buy_order_book, difference)
if difference >= threshold:
min_volume = determine_minimum_volume(sell_order_book, buy_order_book,
balance_state)
min_volume = adjust_minimum_volume_by_trading_cap(deal_cap, min_volume)
min_volume = adjust_maximum_volume_by_trading_cap(deal_cap, min_volume)
min_volume = round_volume_by_exchange_rules(
sell_order_book.exchange_id, buy_order_book.exchange_id,
min_volume, sell_order_book.pair_id)
if min_volume <= 0:
log_arbitrage_determined_volume_not_enough(sell_order_book,
buy_order_book,
msg_queue)
return deal_status
sell_price = adjust_price_by_order_book(sell_order_book.bid,
min_volume)
arbitrage_id = get_next_arbitrage_id()
create_time = get_now_seconds_utc()
trade_at_first_exchange = Trade(DEAL_TYPE.SELL,
sell_order_book.exchange_id,
sell_order_book.pair_id,
sell_price,
min_volume,
sell_order_book.timest,
create_time,
arbitrage_id=arbitrage_id)
buy_price = adjust_price_by_order_book(buy_order_book.ask, min_volume)
trade_at_second_exchange = Trade(DEAL_TYPE.BUY,
buy_order_book.exchange_id,
buy_order_book.pair_id,
buy_price,
min_volume,
buy_order_book.timest,
create_time,
arbitrage_id=arbitrage_id)
final_difference = get_change(sell_price, buy_price, provide_abs=False)
if final_difference <= 0.2:
log_arbitrage_determined_price_not_enough(
sell_price, sell_order_book.bid[FIRST].price, buy_price,
buy_order_book.ask[LAST].price, difference, final_difference,
sell_order_book.pair_id, msg_queue)
return deal_status
trade_pair = TradePair(trade_at_first_exchange,
trade_at_second_exchange,
sell_order_book.timest, buy_order_book.timest,
type_of_deal)
placement_status = action_to_perform(trade_pair, final_difference,
"history_trades.log", worker_pool,
msg_queue)
# NOTE: if we can't update balance for more than TIMEOUT seconds arbitrage process will exit
# for exchange_id in [trade_pair.deal_1.exchange_id, trade_pair.deal_2.exchange_id]:
# update_balance_by_exchange(exchange_id)
# deal_status = placement_status, trade_pair
return deal_status
