def on_message_handler(self, instmt, message):
"""
Incoming message handler
:param instmt: Instrument
:param message: Message
"""
if 'ping' in message:
#handle ping response
ts = message['ping']
self.api_socket.send(json.dumps({'pong': ts}))
elif 'ch' in message:
if 'trade.detail' in message['ch']:
trades = self.api_socket.parse_trade(instmt,
message['tick']['data'])
for trade in trades:
if trade.trade_id != instmt.get_exch_trade_id():
instmt.incr_trade_id()
instmt.set_exch_trade_id(trade.trade_id)
self.insert_trade(instmt, trade)
elif 'depth.step' in message['ch']:
instmt.set_prev_l2_depth(instmt.get_l2_depth().copy())
self.api_socket.parse_l2_depth(instmt, message['tick'])
if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()):
instmt.incr_order_book_id()
self.insert_order_book(instmt)
else:
Logger.error(self.__class__.__name__, 'Not Trade or Market')
def insert_data_worker(self):
while True:
time.sleep(0.5)
if not self.q.empty():
items = []
i = 0
while not self.q.empty():
items.append(self.q.get())
self.q.task_done()
i += 1
self.lock.acquire()
try:
rv = self.client.write_points(items,
database=self.dbname,
time_precision='u')
if not rv:
Logger.error(self.__class__.__name__,
"Error writing to InfluxDb")
except Exception as e:
Logger.error(self.__class__.__name__,
"InfluxDb error: %s" % (e))
self.lock.release()
else:
pass
def on_message_handler(self, instmt, message):
"""
Incoming message handler
:param instmt: Instrument
:param message: Message
"""
keys = message.keys()
if 'info' in keys:
Logger.info(self.__class__.__name__, message['info'])
elif 'subscribe' in keys:
Logger.info(self.__class__.__name__, 'Subscription of %s is %s' % \
(message['request']['args'], \
'successful' if message['success'] else 'failed'))
elif 'table' in keys:
if message['table'] == 'trade':
for trade_raw in message['data']:
if trade_raw["symbol"] == instmt.get_instmt_code():
# Filter out the initial subscriptions
trade = self.api_socket.parse_trade(instmt, trade_raw)
if trade.trade_id != instmt.get_exch_trade_id():
instmt.incr_trade_id()
instmt.set_exch_trade_id(trade.trade_id)
self.insert_trade(instmt, trade)
elif message['table'] == 'orderBookL2':
l2_depth = self.api_socket.parse_l2_depth(instmt, message)
if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()):
instmt.set_prev_l2_depth(instmt.get_l2_depth())
instmt.set_l2_depth(l2_depth)
instmt.incr_order_book_id()
self.insert_order_book(instmt)
else:
Logger.info(self.__class__.__name__, json.dumps(message,indent=2))
else:
Logger.error(self.__class__.__name__, "Unrecognised message:\n" + json.dumps(message))
def on_message_handler(self, instmt, message):
"""
Incoming message handler
:param instmt: Instrument
:param message: Message
"""
keys = message.keys()
if 'info' in keys:
Logger.info(self.__class__.__name__, message['info'])
elif 'subscribe' in keys:
Logger.info(self.__class__.__name__, 'Subscription of %s is %s' % \
(message['request']['args'], \
'successful' if message['success'] else 'failed'))
elif 'table' in keys:
if message['table'] == 'trade':
for trade_raw in message['data']:
if trade_raw["symbol"] == instmt.get_instmt_code():
# Filter out the initial subscriptions
trade = self.api_socket.parse_trade(instmt, trade_raw)
if trade.trade_id != instmt.get_exch_trade_id():
instmt.incr_trade_id()
instmt.set_exch_trade_id(trade.trade_id)
self.insert_trade(instmt, trade)
elif message['table'] == 'orderBook10':
for data in message['data']:
if data["symbol"] == instmt.get_instmt_code():
instmt.set_prev_l2_depth(instmt.get_l2_depth().copy())
self.api_socket.parse_l2_depth(instmt, data)
if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()):
instmt.incr_order_book_id()
self.insert_order_book(instmt)
else:
Logger.info(self.__class__.__name__, json.dumps(message,indent=2))
else:
Logger.error(self.__class__.__name__, "Unrecognised message:\n" + json.dumps(message))
def get_order_book_worker(self, instmt):
"""
Get order book worker
:param instmt: Instrument
"""
while True:
ExchGwQuoine.last_query_time_lock.acquire()
if datetime.now() - ExchGwQuoine.last_query_time < timedelta(
seconds=ExchGwQuoine.waiting_seconds):
ExchGwQuoine.last_query_time_lock.release()
time.sleep(random.uniform(0, 1))
else:
ExchGwQuoine.last_query_time = datetime.now()
try:
l2_depth = self.api_socket.get_order_book(instmt)
if l2_depth is not None and l2_depth.is_diff(
instmt.get_l2_depth()):
instmt.set_prev_l2_depth(instmt.get_l2_depth())
instmt.set_l2_depth(l2_depth)
instmt.incr_order_book_id()
self.insert_order_book(instmt)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in order book: %s" % e)
ExchGwQuoine.last_query_time_lock.release()
def get_trades_worker(self, instmt):
"""
Get order book worker thread
:param instmt: Instrument name
"""
while True:
try:
ret = self.api_socket.get_trades(instmt)
if ret is None or len(ret) == 0:
time.sleep(1)
continue
except Exception as e:
Logger.error(self.__class__.__name__, "Error in trades: %s" % e)
for trade in ret:
assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id)
assert isinstance(instmt.get_exch_trade_id(), str), \
"instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id())
if int(trade.trade_id) > int(instmt.get_exch_trade_id()):
instmt.set_exch_trade_id(trade.trade_id)
instmt.incr_trade_id()
self.insert_trade(instmt, trade)
# After the first time of getting the trade, indicate the instrument
# is recovered
if not instmt.get_recovered():
instmt.set_recovered(True)
time.sleep(1)
def send_order(amount, source, symbol, _type, price=0):
"""
:param amount:
:param source:
:param symbol:
:param _type: 可选值 {buy-market：市价买, sell-market：市价卖, buy-limit：限价买, sell-limit：限价卖}
:param price:
:return:
"""
try:
accounts = get_accounts()
acct_id = accounts['data'][0]['id']
except BaseException as e:
#xie begin , #print 'get acct_id error.%s'%e
Logger.error("HuobiService", "Unrecognised message:\n" % e)
#xie end
acct_id = hbu.ACCOUNT_ID
params = {
"account-id": acct_id,
"amount": amount,
"symbol": symbol,
"type": _type,
"source": source
}
if price:
params["price"] = price
url = '/v1/order/orders/place'
return hbu.api_key_post(params, url)
def get_trades_worker(self, instmt):
"""
Get order book worker thread
:param instmt: Instrument name
"""
while True:
try:
ret = self.api_socket.get_trades(instmt)
if ret is None or len(ret) == 0:
time.sleep(1)
continue
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in trades: %s" % e)
time.sleep(1)
continue
for trade in ret:
assert isinstance(trade.trade_id,
str), "trade.trade_id(%s) = %s" % (type(
trade.trade_id), trade.trade_id)
assert isinstance(instmt.get_exch_trade_id(), str), \
"instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id())
if int(trade.trade_id) > int(instmt.get_exch_trade_id()):
instmt.set_exch_trade_id(trade.trade_id)
instmt.incr_trade_id()
self.insert_trade(instmt, trade)
# After the first time of getting the trade, indicate the instrument
# is recovered
if not instmt.get_recovered():
instmt.set_recovered(True)
time.sleep(1)
def create(self, table, columns, types, primary_key_index=[], is_ifnotexists=True):
"""
Create table in the database.
Caveat - Assign the first few column as the keys!!!
:param table: Table name
:param columns: Column array
:param types: Type array
:param is_ifnotexists: Create table if not exists keyword
"""
if len(columns) != len(types):
raise Exception("Incorrect create statement. Number of columns and that of types are different.\n%s\n%s" % \
(columns, types))
if is_ifnotexists:
ret = self.conn("\\v")
if ret is not None:
for t in ret:
if table == self.decode_qtypes(t):
Logger.info(self.__class__.__name__, "Table %s has been created." % table)
return True
Logger.info(self.__class__.__name__, "Table %s is going to be created." % table)
c = columns[:]
for i in range(0, len(types)):
t = self.convert_type(types[i])
if t is str:
if columns[i].find('date_time') > -1:
c[i] += ":`timestamp$()"
else:
c[i] += ":`symbol$()"
elif t is float:
c[i] += ":`float$()"
elif t is int:
c[i] += ":`long$()"
keys = []
for i in primary_key_index:
keys.append(c[i])
for i in sorted(primary_key_index, reverse=True):
del c[i]
if len(keys) > 0:
command = '%s:([%s] %s)' % (table, '; '.join(keys), '; '.join(c))
else:
command = '%s:(%s)' % (table, '; '.join(c))
self.lock.acquire()
try:
self.conn.sync(command)
except Exception as e:
Logger.error(self.__class__.__name__, "Error in creat statement(%s).\n%s" % (command, e))
finally:
self.lock.release()
return True
def insert(self,
table,
columns,
types,
values,
primary_key_index=[],
is_orreplace=False,
is_commit=True):
"""
Insert into the table
:param table: Table name
:param columns: Column array
:param types: Type array
:param values: Value array
:param primary_key_index: An array of indices of primary keys in columns,
e.g. [0] means the first column is the primary key
:param is_orreplace: Indicate if the query is "INSERT OR REPLACE"
"""
if is_orreplace:
# skip snaphosts
return True
exchange_name, tick = self.split_table_name(table)
if len(columns) != len(values):
return False
idata = {"measurement": tick, "tags": {"exchange": exchange_name}}
fields = {}
for i in range(0, len(columns)):
if columns[i] in self.column_time_name:
idata["time"] = int(self.time_to_unix(values[i]))
# store only trades (skip orders)
if self.column_update_type in columns[i]:
if int(values[i]) is not 2:
return True
if columns[i] in self.column_ignore:
continue
else:
if "int" in types[i]:
fields[columns[i]] = int(values[i])
elif "decimal" in types[i]:
fields[columns[i]] = float(values[i])
elif "varchar" in types[i]:
fields[columns[i]] = values[i]
else:
Logger.error(self.__class__.__name__,
"Unkown value type:{}".format(types[i]))
return True
idata["fields"] = fields
self.q.put(idata)
return True
def on_message_handler(self, instmt, message):
"""
Incoming message handler
:param instmt: Instrument
:param message: Message
"""
if not message:
return
keys = message.keys()
if "bids" in keys:
self.order_book = self.api_socket.parse_l2_depth(instmt, message)
# Insert only if the first 5 levels are different
if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()):
instmt.incr_order_book_id()
self.insert_order_book(instmt)
elif "create_update" in keys:
if message['create_update']:
message['create_update'].update(
{"timestamp": message['timestamp']})
self.api_socket.parse_l2_depth(instmt,
message['create_update'])
# Insert only if the first 5 levels are different
if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()):
instmt.incr_order_book_id()
self.insert_order_book(instmt)
elif message['delete_update']:
message['delete_update'].update(
{"timestamp": message['timestamp']})
self.api_socket.parse_l2_depth(instmt,
message['delete_update'])
# Insert only if the first 5 levels are different
if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()):
instmt.incr_order_book_id()
self.insert_order_book(instmt)
elif message['trade_updates']:
for new_trade in message['trade_updates']:
new_trade.update({"timestamp": message['timestamp']})
trade = self.api_socket.parse_trade(instmt, new_trade)
self.api_socket.parse_l2_depth(instmt, new_trade)
if trade.trade_id != instmt.get_exch_trade_id():
instmt.incr_trade_id()
instmt.set_exch_trade_id(trade.trade_id)
self.insert_trade(instmt, trade)
else:
Logger.error(self.__class__.__name__,
"Unrecognised message:\n" + json.dumps(message))
def get_order_book(cls, instmt):
"""
Get order book
:param instmt: Instrument
:return: Object L2Depth
"""
res = cls.request(cls.get_order_book_link(instmt))
if len(res) > 0 and 'error' in res and len(res['error']) == 0:
res = list(res['result'].values())[0]
return cls.parse_l2_depth(instmt=instmt, raw=res)
else:
Logger.error(cls.__name__,
"Cannot parse the order book. Return:\n%s" % res)
return None
def get_order_book(cls, instmt):
"""
Get order book
:param instmt: Instrument
:return: Object L2Depth
"""
res = cls.request(cls.get_order_book_link(instmt))
if len(res) > 0 and 'error' in res and len(res['error']) == 0:
res = list(res['result'].values())[0]
return cls.parse_l2_depth(instmt=instmt,
raw=res)
else:
Logger.error(cls.__name__, "Cannot parse the order book. Return:\n%s" % res)
return None
def get_order_book_worker(self, instmt):
"""
Get order book worker
:param instmt: Instrument
"""
while True:
try:
l2_depth = self.api_socket.get_order_book(instmt)
if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()):
instmt.set_prev_l2_depth(instmt.get_l2_depth())
instmt.set_l2_depth(l2_depth)
instmt.incr_order_book_id()
self.insert_order_book(instmt)
except Exception as e:
Logger.error(self.__class__.__name__, "Error in order book: %s" % e)
time.sleep(1)
def get_order_book_worker(self, instmt):
"""
Get order book worker
:param instmt: Instrument
"""
while True:
try:
l2_depth = self.api_socket2.get_order_book(instmt)
if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()):
instmt.set_prev_l2_depth(instmt.get_l2_depth())
instmt.set_l2_depth(l2_depth)
instmt.incr_order_book_id()
self.insert_order_book(instmt)
except Exception as e:
Logger.error(self.__class__.__name__, "Error in order book: %s" % e)
time.sleep(1)
def insert(self,
table,
columns,
types,
values,
primary_key_index=[],
is_orreplace=False,
is_commit=True):
"""
Insert into the table
:param table: Table name
:param columns: Column array
:param types: Type array
:param values: Value array
:param primary_key_index: An array of indices of primary keys in columns,
e.g. [0] means the first column is the primary key
:param is_orreplace: Indicate if the query is "INSERT OR REPLACE"
"""
ret = dict(zip(columns, values))
ret['table'] = table
self.lock.acquire()
# print(ret)
# print('columns:', columns)
# print('values:', values)
future = self.conn.send(table, value=ret)
result = True
# Block for 'synchronous' sends
try:
record_metadata = future.get(timeout=60)
# print(record_metadata)
Logger.info(
self.__class__.__name__, "topic: %s, offset: %s" %
(record_metadata.topic, record_metadata.offset))
except Exception as ex:
Logger.error(self.__class__.__name__,
"exception in producer:%s" % ex)
# traceback.print_exc()
result = False
# raise Exception("kafka send failed.")
finally:
self.lock.release()
return result
def get_balance(acct_id=None):
"""
:param acct_id
:return:
"""
if not acct_id:
try:
accounts = get_accounts()
acct_id = ACCOUNT_ID = accounts['data'][0]['id']
except BaseException as e:
#xie begin ,print 'get acct_id error.%s'%e
Logger.error("HuobiService", "Unrecognised message:\n" % e)
#xie end
acct_id = ACCOUNT_ID
url = "/v1/account/accounts/{0}/balance".format(acct_id)
params = {"account-id": acct_id}
return hbu.api_key_get(params, url)
def get_trades_worker(self, instmt):
"""
Get order book worker thread
:param instmt: Instrument name
"""
while True:
ExchGwQuoine.last_query_time_lock.acquire()
if datetime.now() - ExchGwQuoine.last_query_time < timedelta(
seconds=ExchGwQuoine.waiting_seconds):
ExchGwQuoine.last_query_time_lock.release()
time.sleep(random.uniform(0, 1))
else:
ExchGwQuoine.last_query_time = datetime.now()
try:
ret = self.api_socket.get_trades(instmt)
if ret is None or len(ret) == 0:
ExchGwQuoine.last_query_time_lock.release()
continue
for trade in ret:
assert isinstance(
trade.trade_id,
str), "trade.trade_id(%s) = %s" % (type(
trade.trade_id), trade.trade_id)
assert isinstance(instmt.get_exch_trade_id(), str), \
"instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id())
if int(trade.trade_id) > int(
instmt.get_exch_trade_id()):
instmt.set_exch_trade_id(trade.trade_id)
instmt.incr_trade_id()
self.insert_trade(instmt, trade)
# After the first time of getting the trade, indicate the instrument
# is recovered
if not instmt.get_recovered():
instmt.set_recovered(True)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in trades: %s" % e)
ExchGwQuoine.last_query_time_lock.release()
def get_order_book_worker(self, instmt):
"""
Get order book worker
:param instmt: Instrument
"""
while True:
ExchGwQuoine.last_query_time_lock.acquire()
if datetime.now() - ExchGwQuoine.last_query_time < timedelta(seconds=ExchGwQuoine.waiting_seconds):
ExchGwQuoine.last_query_time_lock.release()
time.sleep(random.uniform(0, 1))
else:
ExchGwQuoine.last_query_time = datetime.now()
try:
l2_depth = self.api_socket.get_order_book(instmt)
if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()):
instmt.set_prev_l2_depth(instmt.get_l2_depth())
instmt.set_l2_depth(l2_depth)
instmt.incr_order_book_id()
self.insert_order_book(instmt)
except Exception as e:
Logger.error(self.__class__.__name__, "Error in order book: %s" % e)
ExchGwQuoine.last_query_time_lock.release()
def get_trades_worker(self, instmt):
"""
Get order book worker thread
:param instmt: Instrument name
"""
instmt.set_recovered(False)
while True:
try:
ret = self.api_socket.get_trades(instmt)
for trade in ret:
instmt.incr_trade_id()
self.insert_trade(instmt, trade)
# After the first time of getting the trade, indicate the instrument
# is recovered
if not instmt.get_recovered():
instmt.set_recovered(True)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in trades: %s\nReturn: %s" % (e, ret))
time.sleep(0.5)
def get_trades_worker(self, instmt):
"""
Get order book worker thread
:param instmt: Instrument name
"""
instmt.set_recovered(False)
while True:
try:
ret = self.api_socket.get_trades(instmt)
for trade in ret:
instmt.incr_trade_id()
self.insert_trade(instmt, trade)
# After the first time of getting the trade, indicate the instrument
# is recovered
if not instmt.get_recovered():
instmt.set_recovered(True)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in trades: %s\nReturn: %s" % (e, ret))
time.sleep(0.5)
def get_trades_worker(self, instmt):
"""
Get order book worker thread
:param instmt: Instrument name
"""
while True:
ExchGwQuoine.last_query_time_lock.acquire()
if datetime.now() - ExchGwQuoine.last_query_time < timedelta(seconds=ExchGwQuoine.waiting_seconds):
ExchGwQuoine.last_query_time_lock.release()
time.sleep(random.uniform(0, 1))
else:
ExchGwQuoine.last_query_time = datetime.now()
try:
ret = self.api_socket.get_trades(instmt)
if ret is None or len(ret) == 0:
ExchGwQuoine.last_query_time_lock.release()
continue
for trade in ret:
assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id)
assert isinstance(instmt.get_exch_trade_id(), str), \
"instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id())
if int(trade.trade_id) > int(instmt.get_exch_trade_id()):
instmt.set_exch_trade_id(trade.trade_id)
instmt.incr_trade_id()
self.insert_trade(instmt, trade)
# After the first time of getting the trade, indicate the instrument
# is recovered
if not instmt.get_recovered():
instmt.set_recovered(True)
except Exception as e:
Logger.error(self.__class__.__name__, "Error in trades: %s" % e)
ExchGwQuoine.last_query_time_lock.release()
def create(self,
table,
columns,
types,
primary_key_index=(),
is_ifnotexists=True):
"""
Create table in the database.
Caveat - Assign the first few column as the keys!!!
:param table: Table name
:param columns: Column array
:param types: Type array
:param is_ifnotexists: Create table if not exists keyword
"""
if len(columns) != len(types):
raise Exception("Incorrect create statement. Number of columns and that of types are different.\n%s\n%s" % \
(columns, types))
if is_ifnotexists:
ret = self.conn("\\v")
if ret is not None:
for t in ret:
if table == self.decode_qtypes(t):
Logger.info(self.__class__.__name__,
"Table %s has been created." % table)
return True
Logger.info(self.__class__.__name__,
"Table %s is going to be created." % table)
c = columns[:]
for i in range(0, len(types)):
t = self.convert_type(types[i])
if t is str:
if columns[i].find('date_time') > -1:
c[i] += ":`timestamp$()"
else:
c[i] += ":`symbol$()"
elif t is float:
c[i] += ":`float$()"
elif t is int:
c[i] += ":`long$()"
keys = []
for i in primary_key_index:
keys.append(c[i])
for i in sorted(primary_key_index, reverse=True):
del c[i]
if len(keys) > 0:
command = '%s:([%s] %s)' % (table, '; '.join(keys), '; '.join(c))
else:
command = '%s:(%s)' % (table, '; '.join(c))
self.lock.acquire()
try:
self.conn.sync(command)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in creat statement(%s).\n%s" % (command, e))
finally:
self.lock.release()
return True
def __on_error(self, ws, error):
Logger.error(self.__class__.__name__, "Socket <%s> error:\n %s" % (self.id, error))
if len(self.on_error_handlers) > 0:
for handler in self.on_error_handlers:
handler(ws, error)
