def on_public(self, compressed_data):
msg = process_message(compressed_data)
# FIXME Howdy DK - is this check promissing FAST?
if not self.order_book_is_received and "orderBook" in compressed_data:
self.order_book_is_received = True
order_book_delta = parse_socket_order_book_poloniex(
msg, self.pair_id)
else:
order_book_delta = parse_socket_update_poloniex(msg)
if order_book_delta is None:
#
# Poloniex tend to send heartbeat messages: [1010]
# When no messages have been sent out for one second, the server will send a heartbeat message as follows.
# Absence of heartbeats indicates a protocol or networking issue and the client application is expected
# to close the socket and try again.
#
str_msg = str(msg)
if "1010" in str_msg:
self.last_heartbeat_ts = get_now_seconds_utc()
else:
err_msg = "Poloniex - cant parse update from message: {msg}".format(
msg=str_msg)
log_to_file(err_msg, SOCKET_ERRORS_LOG_FILE_NAME)
else:
self.last_heartbeat_ts = get_now_seconds_utc()
self.on_update(EXCHANGE.POLONIEX, order_book_delta)
def test_time_epoch():
t = get_now_seconds_utc()
t1 = get_now_seconds_local()
t2 = generate_nonce()
print "utc", t
print "local", t1
print "nonce", t2
def check_deal_placements():
if not YES_I_KNOW_WHAT_AM_I_DOING:
die_hard("check_deal_placements may issue a real trade!")
create_time = get_now_seconds_utc()
fake_order_book_time1 = -10
fake_order_book_time2 = -20
deal_volume = 5
pair_id = CURRENCY_PAIR.BTC_TO_ARDR
sell_exchange_id = EXCHANGE.POLONIEX
buy_exchange_id = EXCHANGE.BITTREX
difference = "difference is HUGE"
file_name = "test.log"
msg_queue = get_message_queue()
processor = ConnectionPool(pool_size=2)
trade_at_first_exchange = Trade(DEAL_TYPE.SELL, sell_exchange_id, pair_id,
0.00000001, deal_volume,
fake_order_book_time1, create_time)
trade_at_second_exchange = Trade(DEAL_TYPE.BUY, buy_exchange_id, pair_id,
0.00004, deal_volume,
fake_order_book_time2, create_time)
trade_pairs = TradePair(trade_at_first_exchange, trade_at_second_exchange,
fake_order_book_time1, fake_order_book_time2,
DEAL_TYPE.DEBUG)
init_deals_with_logging_speedy(trade_pairs, difference, file_name,
processor, msg_queue)
def test_trade_history_huobi_methods():
load_keys(API_KEY_PATH)
key = get_key_by_exchange(EXCHANGE.HUOBI)
time_end = get_now_seconds_utc()
time_start = 0 # time_end - POLL_TIMEOUT
pair_name = get_currency_pair_to_huobi(CURRENCY_PAIR.BTC_TO_LSK)
huobi_orders_by_pair = get_order_history_huobi(key, pair_name, time_start,
time_end)
for pair_id in huobi_orders_by_pair:
pair_name = get_currency_pair_to_huobi(pair_id)
print "PAIR NAME: ", pair_name
for b in huobi_orders_by_pair[pair_id]:
print b
res, order_history = get_order_history_huobi(key, pair_name, time_start,
time_end)
if len(order_history) > 0:
for b in order_history:
print b
pg_conn = init_pg_connection(_db_host=DB_HOST,
_db_port=DB_PORT,
_db_name=DB_NAME)
load_recent_huobi_trades_to_db(pg_conn,
time_start,
time_end,
unique_only=True)
def update_min_cap(cfg, deal_cap, processor):
cur_timest_sec = get_now_seconds_utc()
tickers = get_ticker_for_arbitrage(
cfg.pair_id, cur_timest_sec,
[cfg.buy_exchange_id, cfg.sell_exchange_id], processor)
new_cap = compute_new_min_cap_from_tickers(cfg.pair_id, tickers)
if new_cap > 0:
msg = "Updating old cap {op}".format(op=deal_cap)
log_to_file(msg, CAP_ADJUSTMENT_TRACE_LOG_FILE_NAME)
deal_cap.update_min_volume_cap(new_cap, cur_timest_sec)
msg = "New cap {op}".format(op=deal_cap)
log_to_file(msg, CAP_ADJUSTMENT_TRACE_LOG_FILE_NAME)
else:
msg = """CAN'T update minimum_volume_cap for {pair_id} at following
exchanges: {exch1} {exch2}""".format(
pair_id=cfg.pair_id,
exch1=get_exchange_name_by_id(cfg.buy_exchange_id),
exch2=get_exchange_name_by_id(cfg.sell_exchange_id))
print_to_console(msg, LOG_ALL_ERRORS)
log_to_file(msg, cfg.log_file_name)
log_to_file(msg, CAP_ADJUSTMENT_TRACE_LOG_FILE_NAME)
def get_tickers():
all_tickers = {}
timest = get_now_seconds_utc()
bittrex_tickers = {}
for pair_name in BITTREX_CURRENCY_PAIRS:
ticker = get_ticker_bittrex(pair_name, timest)
if ticker is not None:
bittrex_tickers[ticker.pair_id] = ticker
all_tickers[EXCHANGE.BITTREX] = bittrex_tickers
kraken_tickers = {}
for pair_name in KRAKEN_CURRENCY_PAIRS:
ticker = get_ticker_kraken(pair_name, timest)
if ticker is not None:
kraken_tickers[ticker.pair_id] = ticker
all_tickers[EXCHANGE.KRAKEN] = kraken_tickers
huobi_tickers = {}
for pair_name in HUOBI_CURRENCY_PAIRS:
ticker = get_ticker_huobi(pair_name, timest)
if ticker is not None:
huobi_tickers[ticker.pair_id] = ticker
all_tickers[EXCHANGE.HUOBI] = huobi_tickers
# NOTE: poloniex return all tickers by single call
poloniex_tickers = get_tickers_poloniex(POLONIEX_CURRENCY_PAIRS, timest)
all_tickers[EXCHANGE.POLONIEX] = poloniex_tickers
# NOTE: binance return all tickers by single call
binance_tickers = get_tickers_binance(BINANCE_CURRENCY_PAIRS, timest)
all_tickers[EXCHANGE.BINANCE] = binance_tickers
return all_tickers
def get_order_history_huobi(key,
pair_name,
time_start=0,
time_end=get_now_seconds_utc()):
post_details = get_order_history_huobi_post_details(
key, pair_name, time_start, time_end)
err_msg = "get_all_orders_huobi for {pair_name}".format(
pair_name=pair_name)
status_code, json_response = send_get_request_with_header(
post_details.final_url,
post_details.headers,
err_msg,
timeout=HUOBI_DEAL_TIMEOUT)
if get_logging_level() >= LOG_ALL_DEBUG:
msg = "get_order_history_huobi: {sc} {resp}".format(sc=status_code,
resp=json_response)
print_to_console(msg, LOG_ALL_DEBUG)
log_to_file(msg, DEBUG_LOG_FILE_NAME)
historical_orders = []
if status_code == STATUS.SUCCESS:
status_code, historical_orders = get_orders_huobi_result_processor(
json_response, pair_name)
return status_code, historical_orders
def get_order_book_sync_and_slow():
all_order_book = defaultdict(list)
timest = get_now_seconds_utc()
for currency in POLONIEX_CURRENCY_PAIRS:
order_book = get_order_book_poloniex(currency, timest)
if order_book is not None:
all_order_book[EXCHANGE.POLONIEX].append(order_book)
for currency in KRAKEN_CURRENCY_PAIRS:
order_book = get_order_book_kraken(currency, timest)
if order_book is not None:
all_order_book[EXCHANGE.KRAKEN].append(order_book)
for currency in BITTREX_CURRENCY_PAIRS:
order_book = get_order_book_bittrex(currency, timest)
if order_book is not None:
all_order_book[EXCHANGE.BITTREX].append(order_book)
for currency in BINANCE_CURRENCY_PAIRS:
order_book = get_order_book_binance(currency, timest)
if order_book is not None:
all_order_book[EXCHANGE.BINANCE].append(order_book)
for currency in HUOBI_CURRENCY_PAIRS:
order_book = get_order_book_huobi(currency, timest)
if order_book is not None:
all_order_book[EXCHANGE.HUOBI].append(order_book)
return all_order_book
def get_balance_poloniex(key):
"""
https://poloniex.com/tradingApi
{'Key': 'QN6SDFQG-XVG2CGG3-WDDG2WDV-VXZ7MYL3',
'Sign': '368a800fcd4bc0f0d95151ed29c9f84ddf6cae6bc366d3105db1560318da72aa82281b5ea52f4d4ec929dd0eabc7339fe0e7dc824bf0f1c64e099344cd6e74d0'}
{'nonce': 1508507033330, 'command': 'returnCompleteBalances'}
{"LTC":{"available":"5.015","onOrders":"1.0025","btcValue":"0.078"},"NXT:{...} ... }
"""
post_details = get_balance_poloniex_post_details(key)
err_msg = "check poloniex balance called"
timest = get_now_seconds_utc()
error_code, res = send_post_request_with_header(
post_details,
err_msg,
max_tries=POLONIEX_NUM_OF_DEAL_RETRY,
timeout=POLONIEX_DEAL_TIMEOUT)
if error_code == STATUS.SUCCESS:
res = Balance.from_poloniex(timest, res)
return error_code, res
def get_history_time_fast():
end_time = get_now_seconds_utc()
start_time = end_time - POLL_PERIOD_SECONDS
processor = ConnectionPool()
trade_history = get_history_speedup(start_time, end_time, processor)
return trade_history
def test_kraken_trade_history_retrieval(self):
today = get_now_seconds_utc()
yesterday = today - 24 * 3600
for pair_name in KRAKEN_CURRENCIES:
trade_history = get_history_kraken(pair_name, yesterday, today)
for entry in trade_history:
if entry:
self.assertEquals(type(entry), TradeHistory)
def test_poloniex_trade_history_retrieval(self):
today = get_now_seconds_utc()
yesterday = today - 24 * 3600
for pair_name in POLONIEX_CURRENCY_PAIRS:
trade_history = get_history_poloniex(pair_name, yesterday, today)
for entry in trade_history:
if entry:
self.assertEquals(type(entry), TradeHistory)
def test_binance_trade_history_retrieval(self):
today = get_now_seconds_utc()
yesterday = today - 24 * 3600
for pair_name in BINANCE_CURRENCY_PAIRS:
trade_history = get_history_binance(pair_name, yesterday, today)
for entry in trade_history:
if entry:
self.assertEquals(type(entry), TradeHistory)
def test_kraken_ohlc_retrieval(self):
date_end = get_now_seconds_utc()
date_start = date_end - 900
for pair_name in KRAKEN_CURRENCIES:
period = 15
candles = get_ohlc_kraken(pair_name, date_start, date_end, period)
for candle in candles:
if candle:
self.assertEquals(type(candle), Candle)
def test_huobi_ohlc_retrieval(self):
date_end = get_now_seconds_utc()
date_start = date_end - 900
for currency in HUOBI_CURRENCY_PAIRS:
period = "15min"
candles = get_ohlc_huobi(currency, date_start, date_end, period)
for candle in candles:
if candle:
self.assertEquals(type(candle), Candle)
def test_binance_ohlc_retrieval(self):
date_end = get_now_seconds_utc()
date_start = date_end - 900
for currency in BINANCE_CURRENCY_PAIRS:
period = "15m"
candles = get_ohlc_binance(currency, date_start, date_end, period)
for candle in candles:
if candle:
self.assertEquals(type(candle), Candle)
def test_bittrex_ohlc_retrieval(self):
date_end = get_now_seconds_utc()
date_start = date_end - 900
for pair_name in BITTREX_CURRENCY_PAIRS:
period = "thirtyMin"
candles = get_ohlc_bittrex(pair_name, date_start, date_end, period)
for candle in candles:
if candle:
self.assertEquals(type(candle), Candle)
def subscribe(self):
if self.should_run:
die_hard("Poloniex - another subcription thread running?")
if get_logging_level() == LOG_ALL_TRACE:
msg = "Poloniex - call subscribe!"
log_to_file(msg, SOCKET_ERRORS_LOG_FILE_NAME)
print_to_console(msg, LOG_ALL_MARKET_RELATED_CRAP)
self.should_run = True
if get_logging_level() == LOG_ALL_TRACE:
websocket.enableTrace(True)
# Create connection
try:
self.ws = create_connection(POLONIEX_WEBSCOKET_URL,
enable_multithread=True)
self.ws.settimeout(15)
except Exception as e:
msg = 'Poloniex - connect ws error - {}, retry...'.format(str(e))
print_to_console(msg, LOG_ALL_ERRORS)
self.disconnect()
return
# actual subscription in dedicated thread
# self.on_open()
self.ws.send(self.subscribe_string)
log_conect_to_websocket("Poloniex")
# event loop for processing responce
while self.should_run:
try:
compressed_data = self.ws.recv()
self.on_public(compressed_data)
except Exception as e:
log_error_on_receive_from_socket("Poloniex", e)
break
if self.last_heartbeat_ts:
# During last 5 seconds - no heartbeats no any updates
ts_now = get_now_seconds_utc()
if ts_now - self.last_heartbeat_ts > POLONIEX_WEBSOCKET_TIMEOUT:
log_heartbeat_is_missing("Poloniex",
POLONIEX_WEBSOCKET_TIMEOUT,
self.last_heartbeat_ts, ts_now)
break
log_subscription_cancelled("Poloniex")
self.disconnect()
def log_to_file(data, file_name, log_dir=None):
if log_dir is None:
log_dir = get_log_folder()
full_path = os.path.join(log_dir, file_name)
with open(full_path, 'a') as the_file:
ts = get_now_seconds_utc()
pid = os.getpid()
the_file.write("{ts}: PID: {pid} {data}\n".format(ts=ts,
pid=pid,
data=str(data)))
def place_order_by_market_rate(expired_order, msg_queue, priority_queue,
min_volume, balance, order_book, log_file_name):
max_volume = determine_maximum_volume_by_balance(expired_order.pair_id,
expired_order.trade_type,
expired_order.volume,
expired_order.price,
balance)
max_volume = round_volume(expired_order.exchange_id, max_volume,
expired_order.pair_id)
if max_volume < min_volume:
log_too_small_volume(expired_order, max_volume, min_volume, msg_queue)
return
expired_order.volume = max_volume
expired_order.create_time = get_now_seconds_utc()
msg = "Replace EXPIRED order with new one - {tt}".format(tt=expired_order)
err_code, json_document = init_deal(expired_order, msg)
log_expired_order_replacement_result(expired_order, json_document,
msg_queue)
if err_code == STATUS.SUCCESS:
expired_order.execute_time = get_now_seconds_utc()
expired_order.order_book_time = long(order_book.timest)
expired_order.order_id = parse_order_id(expired_order.exchange_id,
json_document)
msg_queue.add_order(ORDERS_MSG, expired_order)
priority_queue.add_order_to_watch_queue(ORDERS_EXPIRE_MSG,
expired_order)
log_placing_new_deal(expired_order, msg_queue, log_file_name)
else:
log_cant_placing_new_deal(expired_order, msg_queue)
msg_queue.add_order(FAILED_ORDERS_MSG, expired_order, log_file_name)
def test_poloniex_ohlc_retrieval(self):
date_end = get_now_seconds_utc()
date_start = date_end - 900
for pair_name in POLONIEX_CURRENCY_PAIRS:
period = 14400
candles = get_ohlc_poloniex(pair_name, date_start, date_end,
period)
for candle in candles:
if candle:
self.assertEquals(type(candle), Candle)
def load_trade_history(args):
"""
Retrieve executed trades from ALL exchanges via REST api
and save into db
Those data later will be used for analysis
of profitability of trading and bot's performance
:param args: period, exchanges, connection details
:return:
"""
pg_conn, settings = process_args(args)
log_initial_settings(
"Starting trade history retrieval for bots using following exchanges: \n",
settings.exchanges)
if args.start_time is None or args.end_time is None:
end_time = get_now_seconds_utc()
start_time = end_time - 24 * 3600
else:
end_time = parse_time(args.end_time, '%Y-%m-%d %H:%M:%S')
start_time = parse_time(args.start_time, '%Y-%m-%d %H:%M:%S')
if start_time == end_time or end_time <= start_time:
die_hard("Wrong time interval provided! {ts0} - {ts1}".format(
ts0=start_time, ts1=end_time))
load_keys(settings.key_path)
while True:
for exchange_id in settings.exchanges:
method = get_trade_retrieval_method_by_exchange(exchange_id)
method(pg_conn, start_time, end_time)
sleep_for(1)
print_to_console("Trade retrieval heartbeat", LOG_ALL_DEBUG)
sleep_for(TRADE_POLL_TIMEOUT)
end_time = get_now_seconds_utc()
start_time = end_time - 24 * 3600
def get_orders_kraken(key):
timest = get_now_seconds_utc()
error_code_1, open_orders = get_open_orders_kraken(key)
error_code_2, closed_orders = get_order_history_kraken(key)
if error_code_1 == STATUS.FAILURE or error_code_2 == STATUS.FAILURE:
return STATUS.FAILURE, None
return STATUS.SUCCESS, OrderState(EXCHANGE.KRAKEN, timest, open_orders,
closed_orders)
def get_order_book(exchange_id, pair_id):
timest = get_now_seconds_utc()
pair_name = get_currency_pair_name_by_exchange_id(pair_id, exchange_id)
if pair_name is None:
print "UNSUPPORTED COMBINATION OF PAIR ID AND EXCHANGE", pair_id, exchange_id
assert pair_name is None
method = get_order_book_method_by_exchange_id(exchange_id)
return method(pair_name, timest)
def get_order_history_poloniex(key, pair_name, time_start=0, time_end=get_now_seconds_utc(), limit=POLONIEX_ORDER_HISTORY_LIMIT):
post_details = get_order_history_poloniex_post_details(key, pair_name, time_start, time_end, limit)
err_msg = "get poloniex order history for time interval for pp={pp}".format(pp=post_details)
status_code, json_document = send_post_request_with_header(post_details, err_msg, max_tries=POLONIEX_NUM_OF_DEAL_RETRY)
historical_orders = []
if status_code == STATUS.SUCCESS:
status_code, historical_orders = get_order_history_poloniex_result_processor(json_document, pair_name)
return status_code, historical_orders
def log_last_balances(exchanges_ids, cache, msg_queue):
timest = get_now_seconds_utc()
ttl = "At ts={ts} what we have at cache".format(ts=timest)
print_to_console(ttl, LOG_ALL_ERRORS)
log_to_file(ttl, "balance.log")
for idx in exchanges_ids:
some_balance = cache.get_balance(idx)
if some_balance is None or (
timest - some_balance.last_update) > BALANCE_EXPIRE_TIMEOUT:
log_warn_balance_not_updating(some_balance, msg_queue)
else:
log_balance_updated(idx, some_balance)
def test_expired_deal_placement():
load_keys(API_KEY_PATH)
priority_queue = get_priority_queue()
ts = get_now_seconds_utc()
order = Trade(DEAL_TYPE.SELL, EXCHANGE.BINANCE, CURRENCY_PAIR.BTC_TO_STRAT, price=0.001, volume=5.0,
order_book_time=ts, create_time=ts, execute_time=ts, order_id='whatever')
msg = "Replace existing order with new one - {tt}".format(tt=order)
err_code, json_document = init_deal(order, msg)
print json_document
order.order_id = parse_order_id(order.exchange_id, json_document)
priority_queue.add_order_to_watch_queue(ORDERS_EXPIRE_MSG, order)
def return_with_no_change(json_document, corresponding_trade):
corresponding_trade.execute_time = get_now_seconds_utc()
try:
corresponding_trade.order_id = dao.parse_order_id(
corresponding_trade.exchange_id, json_document)
except Exception, e:
log_to_file(
"Cant parse order_id! for following document"
"failed with exception - {}".format(e), "parce_order_id.log")
log_to_file(json_document, "parce_order_id.log")
def print_top10(exchange_id, order_book_buy, order_book_sell):
header = "Number of threads: {tn} Last update {ts} {td} from {up_exch_name}\n" \
"\n{buy_exchange}\t\t\t\t\t\t\t\t\t{sell_exchange}\n".\
format(tn=threading.active_count(), ts=get_now_seconds_utc(),
td=ts_to_string_utc(get_now_seconds_utc()),
up_exch_name=get_exchange_name_by_id(exchange_id),
buy_exchange=get_exchange_name_by_id(order_book_buy.exchange_id),
sell_exchange=get_exchange_name_by_id(order_book_sell.exchange_id)
)
os.system('clear')
print(header)
print(BIDS_HEADLINE)
lb1 = len(order_book_buy.bid)
lb2 = len(order_book_sell.bid)
for idx in xrange(0, 10):
if idx < lb1:
print order_book_buy.bid[idx],
else:
print None,
print ORDER_BOOK_DELIMITER,
if idx < lb2:
print order_book_sell.bid[idx]
else:
print None
print(ASKS_HEADLINE)
la1 = len(order_book_buy.ask)
la2 = len(order_book_sell.ask)
for idx in xrange(0, 10):
if idx < la1:
print order_book_buy.ask[idx],
else:
print None
print ORDER_BOOK_DELIMITER,
if idx < la2:
print order_book_sell.ask[idx]
else:
print None
def update_balance(self):
while self.update_balance_run_flag:
cur_timest_sec = get_now_seconds_utc()
self.balance_state = get_updated_balance_arbitrage(cfg, self.balance_state, self.local_cache)
if self.balance_state.expired(cur_timest_sec, self.buy_exchange_id, self.sell_exchange_id,
BALANCE_EXPIRED_THRESHOLD):
log_balance_expired_errors(cfg, self.msg_queue, self.balance_state)
assert False
sleep_for(self.balance_update_timeout)
