def get_order_book_poloniex_result_processor(json_document, pair_name, timest):
if is_error(json_document):
msg = "get_order_book_poloniex_result_processor - error response - {er}".format(
er=json_document)
log_to_file(msg, ERROR_LOG_FILE_NAME)
return None
return OrderBook.from_poloniex(json_document, pair_name, timest)
def get_order_book_huobi_result_processor(json_document, pair_name, timest):
if is_error(json_document) or json_document.get("tick") is None:
msg = "get_order_book_huobi_result_processor - error response - {er}".format(
er=json_document)
log_to_file(msg, ERROR_LOG_FILE_NAME)
return None
return OrderBook.from_huobi(json_document["tick"], pair_name, timest)
def get_order_book_binance(pair_name, timest):
final_url = get_order_book_binance_url(pair_name)
err_msg = "get_order_book_binance called for {pair} at {timest}".format(
pair=pair_name, timest=timest)
error_code, r = send_request(final_url, err_msg)
if error_code == STATUS.SUCCESS and r is not None:
return OrderBook.from_binance(r, pair_name, timest)
return None
def parse_socket_update_huobi(order_book_delta, pair_id):
if "tick" not in order_book_delta:
return None
order_book_delta = order_book_delta["tick"]
sequence_id = long(order_book_delta["version"])
asks = [Deal(price=b[0], volume=b[1]) for b in order_book_delta.get("asks", [])]
bids = [Deal(price=b[0], volume=b[1]) for b in order_book_delta.get("bids", [])]
timest_ms = get_now_seconds_utc_ms()
return OrderBook(pair_id, timest_ms, asks, bids, EXCHANGE.HUOBI, sequence_id)
def get_order_book_kraken_result_processor(json_document, pair_name, timest):
if is_error(json_document):
msg = "get_order_book_kraken_result_processor - error response - {er}".format(
er=json_document)
log_to_file(msg, ERROR_LOG_FILE_NAME)
return None
if pair_name in json_document["result"]:
return OrderBook.from_kraken(json_document["result"][pair_name],
pair_name, timest)
return None
def get_order_book_by_time(pg_conn, timest):
order_books = defaultdict(list)
select_query = "select id, pair_id, exchange_id, timest from order_book where timest = " + str(timest)
cursor = pg_conn.cursor
cursor.execute(select_query)
for row in cursor:
order_book_id = row[0]
order_book_asks = get_order_book_asks(pg_conn, order_book_id)
order_book_bids = get_order_book_bids(pg_conn, order_book_id)
order_books[int(row[2])].append(OrderBook.from_row(row, order_book_asks, order_book_bids))
return order_books
class ArbitrageListener(ArbitrageWrapper):
def __init__(self, cfg, app_settings):
ArbitrageWrapper.__init__(self, cfg)
self._init_infrastructure(app_settings)
def start(self):
self.reset_arbitrage_state()
while True:
if self.buy_subscription.is_running() and self.sell_subscription.is_running():
sleep_for(1)
else:
# We will NOT issue a reset till any pending process still running
while self.buy_subscription.is_running() or self.sell_subscription.is_running():
sleep_for(1)
self.reset_arbitrage_state()
def reset_arbitrage_state(self):
local_timeout = 1
while True:
sleep_for(local_timeout)
log_init_reset()
set_stage(ORDER_BOOK_SYNC_STAGES.RESETTING)
self.update_balance_run_flag = False
self.update_min_cap_run_flag = False
clear_queue(self.sell_exchange_updates)
clear_queue(self.buy_exchange_updates)
self._init_arbitrage_state() # Spawn balance & cap threads, no blocking
self.subscribe_to_order_book_update() # Spawn order book subscription threads, no blocking
self.sync_order_books() # Spawn order book sync threads, BLOCKING till they finished
log_reset_final_stage()
if get_stage() != ORDER_BOOK_SYNC_STAGES.AFTER_SYNC:
self.shutdown_subscriptions()
log_to_file("reset_arbitrage_state - cant sync order book, lets try one more time!", SOCKET_ERRORS_LOG_FILE_NAME)
while self.buy_subscription.is_running() or self.sell_subscription.is_running():
sleep_for(1)
local_timeout += 1
else:
break
log_reset_stage_successfully()
def _init_infrastructure(self, app_settings):
self.priority_queue, self.msg_queue, self.local_cache = init_queues(app_settings)
self.processor = ConnectionPool(pool_size=2)
self.sell_exchange_updates = Queue()
self.buy_exchange_updates = Queue()
buy_subscription_constructor = get_subcribtion_by_exchange(self.buy_exchange_id)
sell_subscription_constructor = get_subcribtion_by_exchange(self.sell_exchange_id)
self.buy_subscription = buy_subscription_constructor(self.pair_id, on_update=self.on_order_book_update)
self.sell_subscription = sell_subscription_constructor(self.pair_id, on_update=self.on_order_book_update)
def _init_arbitrage_state(self):
self.init_deal_cap()
self.init_balance_state()
self.init_order_books()
self.sell_order_book_synced = False
self.buy_order_book_synced = False
set_stage(ORDER_BOOK_SYNC_STAGES.BEFORE_SYNC)
def init_deal_cap(self):
self.update_min_cap_run_flag = True
# TODO FIXME UNCOMMENT self.subscribe_cap_update()
def update_min_cap(self):
log_to_file("Subscribing for updating cap updates", SOCKET_ERRORS_LOG_FILE_NAME)
while self.update_min_cap_run_flag:
update_min_cap(self.cfg, self.deal_cap, self.processor)
for _ in xrange(self.cap_update_timeout):
if self.update_min_cap_run_flag:
sleep_for(1)
log_to_file("Exit from updating cap updates", SOCKET_ERRORS_LOG_FILE_NAME)
def init_balance_state(self):
self.update_balance_run_flag = True
# TODO FIXME UNCOMMENT self.subscribe_balance_update()
def init_order_books(self):
cur_timest_sec = get_now_seconds_utc()
self.order_book_sell = OrderBook(self.pair_id, cur_timest_sec, sell_bids=[], buy_bids=[], exchange_id=self.sell_exchange_id)
self.order_book_buy = OrderBook(self.pair_id, cur_timest_sec, sell_bids=[], buy_bids=[], exchange_id=self.buy_exchange_id)
def update_from_queue(self, exchange_id, order_book, queue):
while True:
if not self.buy_subscription.is_running() or not self.sell_subscription.is_running():
return STATUS.FAILURE
try:
order_book_update = queue.get(block=False)
except:
order_book_update = None
if order_book_update is None:
break
if STATUS.SUCCESS != order_book.update(exchange_id, order_book_update):
return STATUS.FAILURE
queue.task_done()
return STATUS.SUCCESS
def sync_sell_order_book(self):
if self.sell_exchange_id in [EXCHANGE.BINANCE, EXCHANGE.BITTREX]:
self.order_book_sell = get_order_book(self.sell_exchange_id, self.pair_id)
if self.order_book_sell is None:
return
self.order_book_sell.sort_by_price()
if STATUS.FAILURE == self.update_from_queue(self.sell_exchange_id, self.order_book_sell, self.sell_exchange_updates):
self.sell_order_book_synced = False
return
log_finishing_syncing_order_book("SELL")
self.sell_order_book_synced = True
def sync_buy_order_book(self):
if self.buy_exchange_id in [EXCHANGE.BINANCE, EXCHANGE.BITTREX]:
self.order_book_buy = get_order_book(self.buy_exchange_id, self.pair_id)
if self.order_book_buy is None:
return
self.order_book_buy.sort_by_price()
if STATUS.FAILURE == self.update_from_queue(self.buy_exchange_id, self.order_book_buy, self.buy_exchange_updates):
self.buy_order_book_synced = False
return
log_finishing_syncing_order_book("BUY")
self.buy_order_book_synced = True
def sync_order_books(self):
# DK NOTE: Those guys will endup by themselves
msg = "sync_order_books - stage status is {}".format(get_stage())
log_to_file(msg, SOCKET_ERRORS_LOG_FILE_NAME)
sync_sell_order_book_thread = start_process_daemon(self.sync_sell_order_book, args=())
sync_buy_order_book_thread = start_process_daemon(self.sync_buy_order_book, args=())
# Wait for both thread be finished
sync_sell_order_book_thread.join()
sync_buy_order_book_thread.join()
if self.sell_order_book_synced and self.buy_order_book_synced:
set_stage(ORDER_BOOK_SYNC_STAGES.AFTER_SYNC)
log_all_order_book_synced()
def subscribe_cap_update(self):
start_process_daemon(self.update_min_cap, args=())
def update_balance(self):
while self.update_balance_run_flag:
cur_timest_sec = get_now_seconds_utc()
self.balance_state = get_updated_balance_arbitrage(cfg, self.balance_state, self.local_cache)
if self.balance_state.expired(cur_timest_sec, self.buy_exchange_id, self.sell_exchange_id,
BALANCE_EXPIRED_THRESHOLD):
log_balance_expired_errors(cfg, self.msg_queue, self.balance_state)
assert False
sleep_for(self.balance_update_timeout)
def subscribe_balance_update(self):
start_process_daemon(self.update_balance, args=())
def subscribe_to_order_book_update(self):
start_process_daemon(self.buy_subscription.subscribe, args=())
start_process_daemon(self.sell_subscription.subscribe, args=())
def shutdown_subscriptions(self):
self.sell_subscription.disconnect()
self.buy_subscription.disconnect()
def on_order_book_update(self, exchange_id, order_book_updates):
"""
:param exchange_id:
:param order_book_updates: parsed OrderBook or OrderBookUpdates according to exchange specs
:param stage: whether BOTH orderbook synced or NOT
:return:
"""
exchange_name = get_exchange_name_by_id(exchange_id)
print_to_console("Got update for {exch} Current number of threads: {thr_num}"
.format(exch=exchange_name, thr_num=threading.active_count()), LOG_ALL_ERRORS)
current_stage = get_stage()
if not self.buy_subscription.is_running() or not self.sell_subscription.is_running():
log_one_of_subscriptions_failed(self.buy_subscription.is_running(), self.sell_subscription.is_running(), current_stage)
self.shutdown_subscriptions()
return
if order_book_updates is None:
print_to_console("Order book update is NONE! for {}".format(exchange_name), LOG_ALL_ERRORS)
return
if current_stage == ORDER_BOOK_SYNC_STAGES.BEFORE_SYNC:
print_to_console("Syncing in progress ...", LOG_ALL_ERRORS)
if exchange_id == self.buy_exchange_id:
if self.buy_order_book_synced:
order_book_update_status = self.order_book_buy.update(exchange_id, order_book_updates)
if order_book_update_status == STATUS.FAILURE:
log_order_book_update_failed_pre_sync("BUY", exchange_id, order_book_updates)
self.shutdown_subscriptions()
else:
self.buy_exchange_updates.put(order_book_updates)
else:
if self.sell_order_book_synced:
order_book_update_status = self.order_book_sell.update(exchange_id, order_book_updates)
if order_book_update_status == STATUS.FAILURE:
log_order_book_update_failed_pre_sync("SELL", exchange_id, order_book_updates)
self.shutdown_subscriptions()
else:
self.sell_exchange_updates.put(order_book_updates)
elif current_stage == ORDER_BOOK_SYNC_STAGES.AFTER_SYNC:
print_to_console("Update after syncing... {}".format(exchange_name), LOG_ALL_ERRORS)
if exchange_id == self.buy_exchange_id:
order_book_update_status = self.order_book_buy.update(exchange_id, order_book_updates)
if order_book_update_status == STATUS.FAILURE:
log_order_book_update_failed_post_sync(exchange_id, order_book_updates)
self.shutdown_subscriptions()
return
else:
order_book_update_status = self.order_book_sell.update(exchange_id, order_book_updates)
if order_book_update_status == STATUS.FAILURE:
log_order_book_update_failed_post_sync(exchange_id, order_book_updates)
self.shutdown_subscriptions()
return
#
# Remove this line to activate trading
#
print_top10(exchange_id, self.order_book_buy, self.order_book_sell)
if not YES_I_KNOW_WHAT_AM_I_DOING:
die_hard("LIVE TRADING!")
# DK NOTE: only at this stage we are ready for searching for arbitrage
# for mode_id in [DEAL_TYPE.ARBITRAGE, DEAL_TYPE.REVERSE]:
# method = search_for_arbitrage if mode_id == DEAL_TYPE.ARBITRAGE else adjust_currency_balance
# active_threshold = self.threshold if mode_id == DEAL_TYPE.ARBITRAGE else self.reverse_threshold
# FIXME NOTE: order book expiration check
# FIXME NOTE: src dst vs buy sell
ts1 = get_now_seconds_utc_ms()
status_code, deal_pair = search_for_arbitrage(self.order_book_sell, self.order_book_buy,
self.threshold,
self.balance_threshold,
init_deals_with_logging_speedy,
self.balance_state, self.deal_cap,
type_of_deal=DEAL_TYPE.ARBITRAGE,
worker_pool=self.processor,
msg_queue=self.msg_queue)
ts2 = get_now_seconds_utc_ms()
msg = "Start: {ts1} ms End: {ts2} ms Runtime: {d} ms".format(ts1=ts1, ts2=ts2, d=ts2-ts1)
#
# FIXME
#
# Yeah, we write to disk after every trade
# Yeah, it is not really about speed :(
#
log_to_file(msg, "profile.txt")
add_orders_to_watch_list(deal_pair, self.priority_queue)
self.deal_cap.update_max_volume_cap(NO_MAX_CAP_LIMIT)
def init_order_books(self):
cur_timest_sec = get_now_seconds_utc()
self.order_book_sell = OrderBook(self.pair_id, cur_timest_sec, sell_bids=[], buy_bids=[], exchange_id=self.sell_exchange_id)
self.order_book_buy = OrderBook(self.pair_id, cur_timest_sec, sell_bids=[], buy_bids=[], exchange_id=self.buy_exchange_id)
def parse_socket_order_book_poloniex(order_book_snapshot, pair_id):
"""
:param order_book_snapshot:
[
<channel id>,
<sequence number>,
[
[
"i",
{
"currencyPair": "<currency pair name>",
"orderBook": [
{
"<lowest ask price>": "<lowest ask size>",
"<next ask price>": "<next ask size>",
…
},
{
"<highest bid price>": "<highest bid size>",
"<next bid price>": "<next bid size>",
…
}
]
}
]
]
]
order_book_snapshot[2][0][1]["orderBook"][0]
Example:
[
148,
573963482,
[
[
"i",
{
"currencyPair": "BTC_ETH",
"orderBook": [
{
"0.08964203": "0.00225904",
"0.04069708": "15.37598559",
...
},
{
"0.03496358": "0.32591524",
"0.02020000": "0.50000000",
...
}
]
}
]
]
]
:param pair_id:
:return:
"""
timest_ms = get_now_seconds_utc_ms()
sequence_id = long(order_book_snapshot[1])
asks = []
for k, v in order_book_snapshot[2][0][1]["orderBook"][0].iteritems():
asks.append(Deal(k, v))
bids = []
for k, v in order_book_snapshot[2][0][1]["orderBook"][1].iteritems():
bids.append(Deal(k, v))
return OrderBook(pair_id, timest_ms, asks, bids, EXCHANGE.POLONIEX,
sequence_id)
def parse_socket_order_book_bittrex(order_book_snapshot, pair_id):
"""
:param: order_book_snapshot stringified json of following format:
Bittrex order book format:
"S" = "Sells"
"Z" = "Buys"
"M" = "MarketName"
"f" = "Fills"
"N" = "Nonce"
For fills:
"F" = "FillType": FILL | PARTIAL_FILL
"I" = "Id"
"Q" = "Quantity"
"P" = "Price"
"t" = "Total"
"OT" = "OrderType": BUY | SELL
"T" = "TimeStamp"
{
"S": [
{
"Q": 4.29981987,
"R": 0.04083123
},
{
"Q": 0.59844883,
"R": 0.04083824
}],
"Z": [
{
"Q": 10.8408461,
"R": 0.04069406
},
{
"Q": 0.9,
"R": 0.04069405
}],
"M": null,
"f": [
{
"F": "FILL",
"I": 274260522,
"Q": 0.37714445,
"P": 0.04083123,
"t": 0.01539927,
"OT": "BUY",
"T": 1535772645920
},
{
"F": "PARTIAL_FILL",
"I": 274260519,
"Q": 1.75676,
"P": 0.04069406,
"t": 0.07148969,
"OT": "SELL",
"T": 1535772645157
}],
"N": 28964
}
:return: newly assembled OrderBook object
"""
timest_ms = get_now_seconds_utc_ms()
sequence_id = long(order_book_snapshot["N"])
sells = order_book_snapshot["S"]
asks = []
for new_sell in sells:
asks.append(Deal(new_sell["R"], new_sell["Q"]))
buys = order_book_snapshot["Z"]
bids = []
for new_buy in buys:
bids.append(Deal(new_buy["R"], new_buy["Q"]))
# DK WTF NOTE: ignore for now
# fills = order_book_snapshot["f"]
return OrderBook(pair_id, timest_ms, asks, bids, EXCHANGE.BITTREX,
sequence_id)