def cancel_order_binance(key, pair_name, order_id):
body = {
"recvWindow": 5000,
"timestamp": get_now_seconds_utc_ms(),
"symbol": pair_name,
"orderId": order_id
}
post_details = generate_post_request(BINANCE_CANCEL_ORDER, body, key)
if get_logging_level() >= LOG_ALL_MARKET_RELATED_CRAP:
msg = "cancel_order_binance: url - {url} headers - {headers} body - {body}".format(
url=post_details.final_url,
headers=post_details.headers,
body=post_details.body)
print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP)
log_to_file(msg, "market_utils.log")
err_msg = "cancel binance order with id {id}".format(id=order_id)
res = send_delete_request_with_header(post_details, err_msg, max_tries=3)
if get_logging_level() >= LOG_ALL_MARKET_RELATED_CRAP:
print_to_console(res, LOG_ALL_MARKET_RELATED_CRAP)
log_to_file(res, "market_utils.log")
return res
def get_balance_binance_post_details(key):
final_url = BINANCE_CHECK_BALANCE
body = {"timestamp": get_now_seconds_utc_ms(), "recvWindow": 5000}
res = generate_post_request(final_url, body, key)
if should_print_debug():
print_to_console(res, LOG_ALL_MARKET_NETWORK_RELATED_CRAP)
return res
def on_message(ws, msg):
global first_response
name = "poloniex-" + str(get_now_seconds_utc_ms()) + ".json"
with open(name, 'w') as outfile:
j = json.loads(msg)
json.dump(j, outfile)
if "orderBook" in msg and not first_response:
first_response = True
else:
print(msg)
def add_order_to_watch_queue(self, topic_id, order):
"""
Place orders to watch list = priority queue by TIME.
We have to use current time instead of create time of orders to avoid collisions and overwrites.
:param topic_id: Redis key
:param order:
:return:
"""
assert order is not None
return self.r.zadd(topic_id, -get_now_seconds_utc_ms(), pickle.dumps(order))
def parse_socket_update_huobi(order_book_delta, pair_id):
if "tick" not in order_book_delta:
return None
order_book_delta = order_book_delta["tick"]
sequence_id = long(order_book_delta["version"])
asks = [Deal(price=b[0], volume=b[1]) for b in order_book_delta.get("asks", [])]
bids = [Deal(price=b[0], volume=b[1]) for b in order_book_delta.get("bids", [])]
timest_ms = get_now_seconds_utc_ms()
return OrderBook(pair_id, timest_ms, asks, bids, EXCHANGE.HUOBI, sequence_id)
def get_open_orders_binance_post_details(key, pair_name):
body = {
"symbol": pair_name,
"timestamp": get_now_seconds_utc_ms(),
"recvWindow": 5000
}
post_details = generate_post_request(BINANCE_GET_ALL_OPEN_ORDERS, body,
key)
if get_logging_level() >= LOG_ALL_MARKET_RELATED_CRAP:
msg = "get_open_orders_binance_post_details: {res}".format(
res=post_details)
print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP)
log_to_file(msg, "market_utils.log")
return post_details
def parse_socket_update_binance(order_book_delta):
"""
How to manage a local order book correctly
Open a stream to wss://stream.binance.com:9443/ws/bnbbtc@depth
Buffer the events you receive from the stream
Get a depth snapshot from **https://www.binance.com/api/v1/depth?symbol=BNBBTC&limit=1000"
Drop any event where u is <= lastUpdateId in the snapshot
The first processed should have U <= lastUpdateId+1 AND u >= lastUpdateId+1
While listening to the stream, each new event's U should be equal to the previous event's u+1
The data in each event is the absolute quantity for a price level
If the quantity is 0, remove the price level
Receiving an event that removes a price level that is not in your local order book can happen and is normal.
4
"U": 157, // First update ID in event
"u": 160, // Final update ID in event
{"e": "depthUpdate", "E": 1527861613915, "s": "DASHBTC", "U": 45790140, "u": 45790142,
"b": [["0.04073500", "2.02000000", []], ["0.04073200", "0.00000000", []]],
"a": [["0.04085300", "0.00000000", []]]}
:param order_book_delta:
:return:
"""
timest_ms = get_now_seconds_utc_ms()
sequence_id = long(order_book_delta["U"])
sequence_id_end = long(order_book_delta["u"])
asks = [Deal(a[0], a[1]) for a in order_book_delta["a"]]
bids = [Deal(b[0], b[1]) for b in order_book_delta["b"]]
trades_sell = []
trades_buy = []
return OrderBookUpdate(sequence_id, bids, asks, timest_ms, trades_sell,
trades_buy, sequence_id_end)
def from_bittrex(cls, json_document, currency, timest):
"""
{"success":true,"message":"","result":{"buy":[{"Quantity":12.76073322,"Rate":0.01557999},{"Quantity":12.01802925,"Rate":0.01557998}
"sell":[{"Quantity":0.38767680,"Rate":0.01560999},{"Quantity":2.24182363,"Rate":0.01561999}
"""
sell_bids = []
if "sell" in json_document and json_document["sell"] is not None:
for b in json_document["sell"]:
sell_bids.append(Deal(b["Rate"], b["Quantity"]))
buy_bids = []
if "buy" in json_document and json_document["buy"] is not None:
for b in json_document["buy"]:
buy_bids.append(Deal(b["Rate"], b["Quantity"]))
pair_id = get_currency_pair_from_bittrex(currency)
# DK FIXME! not exactly but Rest API doesnt offer any viable options :/
sequence_id = get_now_seconds_utc_ms()
return OrderBook(pair_id, timest, sell_bids, buy_bids,
EXCHANGE.BITTREX, sequence_id)
def get_order_history_binance_post_details(key,
pair_name,
limit,
last_order_id=None):
body = {
"symbol": pair_name,
"limit": limit,
"timestamp": get_now_seconds_utc_ms(),
"recvWindow": 5000
}
if last_order_id is not None:
body["orderId"] = last_order_id
post_details = generate_post_request(BINANCE_GET_ALL_ORDERS, body, key)
if get_logging_level() >= LOG_ALL_MARKET_RELATED_CRAP:
msg = "get orders history binance: {res}".format(res=post_details)
print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP)
log_to_file(msg, "market_utils.log")
return post_details
def get_trades_history_binance(key, pair_name, limit, last_order_id=None):
final_url = BINANCE_GET_ALL_TRADES
body = []
if last_order_id is not None:
body.append(("fromId", last_order_id))
body.append(("symbol", pair_name))
body.append(("limit", limit))
body.append(("timestamp", get_now_seconds_utc_ms()))
body.append(("recvWindow", 5000))
body.append(("signature", signed_body_256(body, key.secret)))
post_details = generate_post_request(final_url, body, key)
if get_logging_level() >= LOG_ALL_DEBUG:
msg = "get_trades_history_binance: {res}".format(res=post_details)
print_to_console(msg, LOG_ALL_DEBUG)
log_to_file(msg, DEBUG_LOG_FILE_NAME)
err_msg = "get_all_trades_binance for {pair_name}".format(
pair_name=pair_name)
error_code, res = send_get_request_with_header(
post_details.final_url,
post_details.headers,
err_msg,
timeout=BINANCE_DEAL_TIMEOUT)
if get_logging_level() >= LOG_ALL_DEBUG:
msg = "get_all_trades_binance: {er_c} {r}".format(er_c=error_code,
r=res)
print_to_console(msg, LOG_ALL_DEBUG)
log_to_file(msg, DEBUG_LOG_FILE_NAME)
return error_code, res
def add_buy_order_binance_url(key, pair_name, price, amount):
# curl -H "X-MBX-APIKEY: vmPUZE6mv9SD5VNHk4HlWFsOr6aKE2zvsw0MuIgwCIPy6utIco14y7Ju91duEh8A"
# -X POST 'https://api.binance.com/api/v3/order' -d 'symbol=LTCBTC&side=BUY&type=LIMIT&timeInForce=GTC&quantity=1
# &price=0.1&recvWindow=6000000×tamp=1499827319559&signature=c8db56825ae71d6d79447849e617115f4a920fa2acdcab2b053c4b2838bd6b71'
body = {
"symbol": pair_name,
"side": "BUY",
"type": "LIMIT",
"timeInForce": "GTC",
"recvWindow": 5000,
"timestamp": get_now_seconds_utc_ms(),
"quantity": amount,
"price": float_to_str(price)
}
res = generate_post_request(BINANCE_BUY_ORDER, body, key)
if get_logging_level() >= LOG_ALL_MARKET_RELATED_CRAP:
msg = "add_buy_order_binance: {res}".format(res=res)
print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP)
log_to_file(msg, "market_utils.log")
return res
def add_sell_order_binance_url(key, pair_name, price, amount):
final_url = BINANCE_SELL_ORDER
body = {
"symbol": pair_name,
"side": "SELL",
"type": "LIMIT",
"timeInForce": "GTC",
"recvWindow": 5000,
"timestamp": get_now_seconds_utc_ms(),
"quantity": amount,
"price": float_to_str(price)
}
res = generate_post_request(final_url, body, key)
if get_logging_level() >= LOG_ALL_MARKET_RELATED_CRAP:
msg = "add_sell_order_binance: url - {url} headers - {headers} body - {body}".format(
url=res.final_url, headers=res.headers, body=res.body)
print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP)
log_to_file(msg, "market_utils.log")
return res
def parse_socket_order_book_bittrex(order_book_snapshot, pair_id):
"""
:param: order_book_snapshot stringified json of following format:
Bittrex order book format:
"S" = "Sells"
"Z" = "Buys"
"M" = "MarketName"
"f" = "Fills"
"N" = "Nonce"
For fills:
"F" = "FillType": FILL | PARTIAL_FILL
"I" = "Id"
"Q" = "Quantity"
"P" = "Price"
"t" = "Total"
"OT" = "OrderType": BUY | SELL
"T" = "TimeStamp"
{
"S": [
{
"Q": 4.29981987,
"R": 0.04083123
},
{
"Q": 0.59844883,
"R": 0.04083824
}],
"Z": [
{
"Q": 10.8408461,
"R": 0.04069406
},
{
"Q": 0.9,
"R": 0.04069405
}],
"M": null,
"f": [
{
"F": "FILL",
"I": 274260522,
"Q": 0.37714445,
"P": 0.04083123,
"t": 0.01539927,
"OT": "BUY",
"T": 1535772645920
},
{
"F": "PARTIAL_FILL",
"I": 274260519,
"Q": 1.75676,
"P": 0.04069406,
"t": 0.07148969,
"OT": "SELL",
"T": 1535772645157
}],
"N": 28964
}
:return: newly assembled OrderBook object
"""
timest_ms = get_now_seconds_utc_ms()
sequence_id = long(order_book_snapshot["N"])
sells = order_book_snapshot["S"]
asks = []
for new_sell in sells:
asks.append(Deal(new_sell["R"], new_sell["Q"]))
buys = order_book_snapshot["Z"]
bids = []
for new_buy in buys:
bids.append(Deal(new_buy["R"], new_buy["Q"]))
# DK WTF NOTE: ignore for now
# fills = order_book_snapshot["f"]
return OrderBook(pair_id, timest_ms, asks, bids, EXCHANGE.BITTREX,
sequence_id)
def on_order_book_update(self, exchange_id, order_book_updates):
"""
:param exchange_id:
:param order_book_updates: parsed OrderBook or OrderBookUpdates according to exchange specs
:param stage: whether BOTH orderbook synced or NOT
:return:
"""
exchange_name = get_exchange_name_by_id(exchange_id)
print_to_console("Got update for {exch} Current number of threads: {thr_num}"
.format(exch=exchange_name, thr_num=threading.active_count()), LOG_ALL_ERRORS)
current_stage = get_stage()
if not self.buy_subscription.is_running() or not self.sell_subscription.is_running():
log_one_of_subscriptions_failed(self.buy_subscription.is_running(), self.sell_subscription.is_running(), current_stage)
self.shutdown_subscriptions()
return
if order_book_updates is None:
print_to_console("Order book update is NONE! for {}".format(exchange_name), LOG_ALL_ERRORS)
return
if current_stage == ORDER_BOOK_SYNC_STAGES.BEFORE_SYNC:
print_to_console("Syncing in progress ...", LOG_ALL_ERRORS)
if exchange_id == self.buy_exchange_id:
if self.buy_order_book_synced:
order_book_update_status = self.order_book_buy.update(exchange_id, order_book_updates)
if order_book_update_status == STATUS.FAILURE:
log_order_book_update_failed_pre_sync("BUY", exchange_id, order_book_updates)
self.shutdown_subscriptions()
else:
self.buy_exchange_updates.put(order_book_updates)
else:
if self.sell_order_book_synced:
order_book_update_status = self.order_book_sell.update(exchange_id, order_book_updates)
if order_book_update_status == STATUS.FAILURE:
log_order_book_update_failed_pre_sync("SELL", exchange_id, order_book_updates)
self.shutdown_subscriptions()
else:
self.sell_exchange_updates.put(order_book_updates)
elif current_stage == ORDER_BOOK_SYNC_STAGES.AFTER_SYNC:
print_to_console("Update after syncing... {}".format(exchange_name), LOG_ALL_ERRORS)
if exchange_id == self.buy_exchange_id:
order_book_update_status = self.order_book_buy.update(exchange_id, order_book_updates)
if order_book_update_status == STATUS.FAILURE:
log_order_book_update_failed_post_sync(exchange_id, order_book_updates)
self.shutdown_subscriptions()
return
else:
order_book_update_status = self.order_book_sell.update(exchange_id, order_book_updates)
if order_book_update_status == STATUS.FAILURE:
log_order_book_update_failed_post_sync(exchange_id, order_book_updates)
self.shutdown_subscriptions()
return
#
# Remove this line to activate trading
#
print_top10(exchange_id, self.order_book_buy, self.order_book_sell)
if not YES_I_KNOW_WHAT_AM_I_DOING:
die_hard("LIVE TRADING!")
# DK NOTE: only at this stage we are ready for searching for arbitrage
# for mode_id in [DEAL_TYPE.ARBITRAGE, DEAL_TYPE.REVERSE]:
# method = search_for_arbitrage if mode_id == DEAL_TYPE.ARBITRAGE else adjust_currency_balance
# active_threshold = self.threshold if mode_id == DEAL_TYPE.ARBITRAGE else self.reverse_threshold
# FIXME NOTE: order book expiration check
# FIXME NOTE: src dst vs buy sell
ts1 = get_now_seconds_utc_ms()
status_code, deal_pair = search_for_arbitrage(self.order_book_sell, self.order_book_buy,
self.threshold,
self.balance_threshold,
init_deals_with_logging_speedy,
self.balance_state, self.deal_cap,
type_of_deal=DEAL_TYPE.ARBITRAGE,
worker_pool=self.processor,
msg_queue=self.msg_queue)
ts2 = get_now_seconds_utc_ms()
msg = "Start: {ts1} ms End: {ts2} ms Runtime: {d} ms".format(ts1=ts1, ts2=ts2, d=ts2-ts1)
#
# FIXME
#
# Yeah, we write to disk after every trade
# Yeah, it is not really about speed :(
#
log_to_file(msg, "profile.txt")
add_orders_to_watch_list(deal_pair, self.priority_queue)
self.deal_cap.update_max_volume_cap(NO_MAX_CAP_LIMIT)
def parse_socket_order_book_poloniex(order_book_snapshot, pair_id):
"""
:param order_book_snapshot:
[
<channel id>,
<sequence number>,
[
[
"i",
{
"currencyPair": "<currency pair name>",
"orderBook": [
{
"<lowest ask price>": "<lowest ask size>",
"<next ask price>": "<next ask size>",
…
},
{
"<highest bid price>": "<highest bid size>",
"<next bid price>": "<next bid size>",
…
}
]
}
]
]
]
order_book_snapshot[2][0][1]["orderBook"][0]
Example:
[
148,
573963482,
[
[
"i",
{
"currencyPair": "BTC_ETH",
"orderBook": [
{
"0.08964203": "0.00225904",
"0.04069708": "15.37598559",
...
},
{
"0.03496358": "0.32591524",
"0.02020000": "0.50000000",
...
}
]
}
]
]
]
:param pair_id:
:return:
"""
timest_ms = get_now_seconds_utc_ms()
sequence_id = long(order_book_snapshot[1])
asks = []
for k, v in order_book_snapshot[2][0][1]["orderBook"][0].iteritems():
asks.append(Deal(k, v))
bids = []
for k, v in order_book_snapshot[2][0][1]["orderBook"][1].iteritems():
bids.append(Deal(k, v))
return OrderBook(pair_id, timest_ms, asks, bids, EXCHANGE.POLONIEX,
sequence_id)
def parse_socket_update_poloniex(order_book_delta):
"""
Message format for ticker
[
1002, Channel
null, Unknown
[
121, CurrencyPairID
"10777.56054438", Last
"10800.00000000", lowestAsk
"10789.20000001", highestBid
"-0.00860373", percentChange
"72542984.79776118", baseVolume
"6792.60163706", quoteVolume
0, isForzen
"11400.00000000", high24hr
"9880.00000009" low24hr
]
]
[1002,null,[158,"0.00052808","0.00053854","0.00052926","0.05571659","4.07923480","7302.01523251",0,"0.00061600","0.00049471"]]
So the columns for orders are
messageType -> t/trade, o/order
tradeID -> only for trades, just a number
orderType -> 1/bid,0/ask
rate
amount
time
sequence
148 is code for BTCETH, yeah there is no documentation.. but when trades occur You can figure out.
Bid is always 1, cause You add something new..
PairId, Nonce, orders\trades deltas:
[24,219199090,[["o",1,"0.04122908","0.01636493"],["t","10026908",0,"0.04122908","0.00105314",1527880700]]]
[24,219201009,[["o",0,"0.04111587","0.00000000"],["o",0,"0.04111174","1.52701255"]]]
[24,219164304,[["o",1,"0.04064791","0.01435233"],["o",1,"0.04068034","0.16858384"]]]
:param order_book_delta:
:return:
"""
asks = []
bids = []
trades_sell = []
trades_buy = []
# We suppose that bid and ask are sorted in particular order:
# for bids - highest - first
# for asks - lowest - first
if len(order_book_delta) < 3:
return None
timest_ms = get_now_seconds_utc_ms()
sequence_id = long(order_book_delta[1])
delta = order_book_delta[2]
for entry in delta:
if entry[0] == POLONIEX_WEBSOCKET_ORDER:
new_deal = Deal(entry[2], entry[3])
# If it is just orders - we insert in a way to keep sorted order
if entry[1] == POLONIEX_WEBSOCKET_ORDER_ASK:
asks.append(new_deal)
elif entry[1] == POLONIEX_WEBSOCKET_ORDER_BID:
bids.append(new_deal)
else:
msg = "Poloniex socket update parsing - {update} total: {ttt}".format(
update=entry, ttt=order_book_delta)
log_to_file(msg, SOCKET_ERRORS_LOG_FILE_NAME)
elif entry[0] == POLONIEX_WEBSOCKET_TRADE:
# FIXME NOTE: this is ugly hack to avoid creation of custom objects
# and at the same Deal object contains lt method that
# used by bisect for efficient binary search in sorted list
new_deal = Deal(entry[3], entry[4])
# For trade - vice-versa we should update opposite arrays:
# in case we have trade with type bid -> we will update orders at ask
# in case we have trade with type ask -> we will update orders at bid
if entry[2] == POLONIEX_WEBSOCKET_ORDER_BID:
trades_sell.append(new_deal)
elif entry[2] == POLONIEX_WEBSOCKET_ORDER_ASK:
trades_buy.append(new_deal)
else:
msg = "Poloniex socket update parsing - {wtf}".format(
wtf=entry)
log_to_file(msg, SOCKET_ERRORS_LOG_FILE_NAME)
else:
msg = "Poloniex socket update parsing - UNKNOWN TYPE - {wtf}".format(
wtf=entry)
log_to_file(msg, SOCKET_ERRORS_LOG_FILE_NAME)
# FIXME NOTE: during update we are not using trades_sell\trades_buy at all - so probably
# better not to use them at all
return OrderBookUpdate(sequence_id, bids, asks, timest_ms, trades_sell,
trades_buy)
def parse_socket_update_bittrex(order_book_delta):
"""
https://bittrex.github.io/#callback-for-1
"S" = "Sells"
"Z" = "Buys"
"Q" = "Quantity"
"R" = "Rate"
"TY" = "Type"
The Type key can be one of the following values: 0 = ADD, 1 = REMOVE, 2 = UPDATE
"M" = "MarketName"
"N" = "Nonce"
"f" = "Fills"
3 {u'S': [],
u'Z': [{u'Q': 0.0, u'R': 0.04040231, u'TY': 1}, {u'Q': 0.78946119, u'R': 0.00126352, u'TY': 0}],
u'M': u'BTC-DASH',
u'f': [],
u'N': 15692}
3 {u'S': [],
u'Z': [{u'Q': 1.59914865, u'R': 0.040436, u'TY': 0}, {u'Q': 0.0, u'R': 0.04040232, u'TY': 1}],
u'M': u'BTC-DASH',
u'f': [],
u'N': 15691}
u'f': [
{u'Q': 0.11299437,
u'R': 0.042135,
u'OT': u'BUY',
u'T': 1527961548500},
{u'Q': 0.39487459, u'R': 0.04213499, u'OT': u'BUY', u'T': 1527961548500}],
:param order_book_delta
:return:
"""
timest_ms = get_now_seconds_utc_ms()
sequence_id = long(order_book_delta["N"])
asks = []
bids = []
trades_sell = []
trades_buy = []
sells = order_book_delta["S"]
buys = order_book_delta["Z"]
fills = order_book_delta["f"]
for new_sell in sells:
new_deal = Deal(new_sell["R"], new_sell["Q"])
if "TY" not in new_sell:
msg = "Bittrex socket update - within SELL array some weird format - no TY - {wtf}".format(
wtf=new_sell)
log_to_file(msg, SOCKET_ERRORS_LOG_FILE_NAME)
continue
type_update = int(new_sell["TY"])
if type_update in [
BittrexParameters.BITTREX_ORDER_ADD,
BittrexParameters.BITTREX_ORDER_UPDATE
]:
asks.append(new_deal)
elif type_update == BittrexParameters.BITTREX_ORDER_REMOVE:
asks.append(Deal(new_sell["R"], 0))
else:
msg = "Bittrex socket un-supported sells format? {wtf}".format(
wtf=new_sell)
log_to_file(msg, SOCKET_ERRORS_LOG_FILE_NAME)
for new_buy in buys:
new_deal = Deal(new_buy["R"], new_buy["Q"])
if "TY" not in new_buy:
msg = "Bittrex socket update - within BUYS array some weird format - no TY - {wtf}".format(
wtf=new_buy)
log_to_file(msg, SOCKET_ERRORS_LOG_FILE_NAME)
continue
type_update = int(new_buy["TY"])
if type_update in [
BittrexParameters.BITTREX_ORDER_ADD,
BittrexParameters.BITTREX_ORDER_UPDATE
]:
bids.append(new_deal)
elif type_update == BittrexParameters.BITTREX_ORDER_REMOVE:
bids.append(Deal(new_buy["R"], 0))
else:
msg = "Bittrex socket un-supported buys format? {wtf}".format(
wtf=new_buy)
log_to_file(msg, SOCKET_ERRORS_LOG_FILE_NAME)
return OrderBookUpdate(sequence_id, bids, asks, timest_ms, trades_sell,
trades_buy)
def add_message(self, topic_id, msg):
msg_with_ts = "{msg}\nTS:{ts}".format(msg=msg, ts=get_now_seconds_utc_ms())
self.r.rpush(topic_id, msg_with_ts)
