def get_order_book_worker(self, instmt):
"""
Get order book worker
:param instmt: Instrument
"""
while True:
ExchGwQuoine.last_query_time_lock.acquire()
if datetime.now() - ExchGwQuoine.last_query_time < timedelta(
seconds=ExchGwQuoine.waiting_seconds):
ExchGwQuoine.last_query_time_lock.release()
time.sleep(random.uniform(0, 1))
else:
ExchGwQuoine.last_query_time = datetime.now()
try:
l2_depth = self.api_socket.get_order_book(instmt)
if l2_depth is not None and l2_depth.is_diff(
instmt.get_l2_depth()):
instmt.set_prev_l2_depth(instmt.get_l2_depth())
instmt.set_l2_depth(l2_depth)
instmt.incr_order_book_id()
self.insert_order_book(instmt)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in order book: %s" % e)
ExchGwQuoine.last_query_time_lock.release()
def get_trades_worker(self, instmt):
"""
Get order book worker thread
:param instmt: Instrument name
"""
while True:
try:
ret = self.api_socket.get_trades(instmt)
if ret is None or len(ret) == 0:
time.sleep(1)
continue
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in trades: %s" % e)
for trade in ret:
assert isinstance(trade.trade_id,
str), "trade.trade_id(%s) = %s" % (type(
trade.trade_id), trade.trade_id)
assert isinstance(instmt.get_exch_trade_id(), str), \
"instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id())
if int(trade.trade_id) > int(instmt.get_exch_trade_id()):
instmt.set_exch_trade_id(trade.trade_id)
instmt.incr_trade_id()
self.insert_trade(instmt, trade)
# After the first time of getting the trade, indicate the instrument
# is recovered
if not instmt.get_recovered():
instmt.set_recovered(True)
time.sleep(1)
def on_message_handler(self, instmt, message):
"""
Incoming message handler
:param instmt: Instrument
:param message: Message
"""
keys = message.keys()
if 'info' in keys:
Logger.info(self.__class__.__name__, message['info'])
elif 'subscribe' in keys:
Logger.info(self.__class__.__name__, 'Subscription of %s is %s' % \
(message['request']['args'], \
'successful' if message['success'] else 'failed'))
elif 'table' in keys:
if message['table'] == 'trade':
for trade_raw in message['data']:
if trade_raw["symbol"] == instmt.get_instmt_code():
# Filter out the initial subscriptions
trade = self.api_socket.parse_trade(instmt, trade_raw)
if trade.trade_id != instmt.get_exch_trade_id():
instmt.incr_trade_id()
instmt.set_exch_trade_id(trade.trade_id)
self.insert_trade(instmt, trade)
elif message['table'] == 'orderBookL2':
l2_depth = self.api_socket.parse_l2_depth(instmt, message)
if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()):
instmt.set_prev_l2_depth(instmt.get_l2_depth())
instmt.set_l2_depth(l2_depth)
instmt.incr_order_book_id()
self.insert_order_book(instmt)
else:
Logger.info(self.__class__.__name__, json.dumps(message,indent=2))
else:
Logger.error(self.__class__.__name__, "Unrecognised message:\n" + json.dumps(message))
def on_message_handler(self, instmt, message):
"""
Incoming message handler
:param instmt: Instrument
:param message: Message
"""
if not message:
return
keys = message.keys()
if "bids" in keys:
self.order_book = self.api_socket.parse_l2_depth(instmt, message)
# Insert only if the first 5 levels are different
if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()):
instmt.incr_order_book_id()
self.insert_order_book(instmt)
elif "create_update" in keys:
if message['create_update']:
message['create_update'].update(
{"timestamp": message['timestamp']})
self.api_socket.parse_l2_depth(instmt,
message['create_update'])
# Insert only if the first 5 levels are different
if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()):
instmt.incr_order_book_id()
self.insert_order_book(instmt)
elif message['delete_update']:
message['delete_update'].update(
{"timestamp": message['timestamp']})
self.api_socket.parse_l2_depth(instmt,
message['delete_update'])
# Insert only if the first 5 levels are different
if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()):
instmt.incr_order_book_id()
self.insert_order_book(instmt)
elif message['trade_updates']:
for new_trade in message['trade_updates']:
new_trade.update({"timestamp": message['timestamp']})
trade = self.api_socket.parse_trade(instmt, new_trade)
self.api_socket.parse_l2_depth(instmt, new_trade)
if trade.trade_id != instmt.get_exch_trade_id():
instmt.incr_trade_id()
instmt.set_exch_trade_id(trade.trade_id)
self.insert_trade(instmt, trade)
else:
Logger.error(self.__class__.__name__,
"Unrecognised message:\n" + json.dumps(message))
def get_order_book(cls, instmt):
"""
Get order book
:param instmt: Instrument
:return: Object L2Depth
"""
res = cls.request(cls.get_order_book_link(instmt))
if len(res) > 0 and 'error' in res and len(res['error']) == 0:
res = list(res['result'].values())[0]
return cls.parse_l2_depth(instmt=instmt,
raw=res)
else:
Logger.error(cls.__name__, "Cannot parse the order book. Return:\n%s" % res)
return None
def get_order_book_worker(self, instmt):
"""
Get order book worker
:param instmt: Instrument
"""
while True:
try:
l2_depth = self.api_socket.get_order_book(instmt)
if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()):
instmt.set_prev_l2_depth(instmt.get_l2_depth())
instmt.set_l2_depth(l2_depth)
instmt.incr_order_book_id()
self.insert_order_book(instmt)
except Exception as e:
Logger.error(self.__class__.__name__, "Error in order book: %s" % e)
time.sleep(1)
def get_trades_worker(self, instmt):
"""
Get order book worker thread
:param instmt: Instrument name
"""
while True:
ExchGwQuoine.last_query_time_lock.acquire()
if datetime.now() - ExchGwQuoine.last_query_time < timedelta(
seconds=ExchGwQuoine.waiting_seconds):
ExchGwQuoine.last_query_time_lock.release()
time.sleep(random.uniform(0, 1))
else:
ExchGwQuoine.last_query_time = datetime.now()
try:
ret = self.api_socket.get_trades(instmt)
if ret is None or len(ret) == 0:
ExchGwQuoine.last_query_time_lock.release()
continue
for trade in ret:
assert isinstance(
trade.trade_id,
str), "trade.trade_id(%s) = %s" % (type(
trade.trade_id), trade.trade_id)
assert isinstance(instmt.get_exch_trade_id(), str), \
"instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id())
if int(trade.trade_id) > int(
instmt.get_exch_trade_id()):
instmt.set_exch_trade_id(trade.trade_id)
instmt.incr_trade_id()
self.insert_trade(instmt, trade)
# After the first time of getting the trade, indicate the instrument
# is recovered
if not instmt.get_recovered():
instmt.set_recovered(True)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in trades: %s" % e)
ExchGwQuoine.last_query_time_lock.release()
def get_trades_worker(self, instmt):
"""
Get order book worker thread
:param instmt: Instrument name
"""
instmt.set_recovered(False)
while True:
try:
ret = self.api_socket.get_trades(instmt)
for trade in ret:
instmt.incr_trade_id()
self.insert_trade(instmt, trade)
# After the first time of getting the trade, indicate the instrument
# is recovered
if not instmt.get_recovered():
instmt.set_recovered(True)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in trades: %s\nReturn: %s" % (e, ret))
time.sleep(0.5)
def create(self,
table,
columns,
types,
primary_key_index=[],
is_ifnotexists=True):
"""
Create table in the database.
Caveat - Assign the first few column as the keys!!!
:param table: Table name
:param columns: Column array
:param types: Type array
:param is_ifnotexists: Create table if not exists keyword
"""
if len(columns) != len(types):
raise Exception("Incorrect create statement. Number of columns and that of types are different.\n%s\n%s" % \
(columns, types))
if is_ifnotexists:
ret = self.conn("\\v")
if ret is not None:
for t in ret:
if table == self.decode_qtypes(t):
Logger.info(self.__class__.__name__,
"Table %s has been created." % table)
return True
Logger.info(self.__class__.__name__,
"Table %s is going to be created." % table)
c = columns[:]
for i in range(0, len(types)):
t = self.convert_type(types[i])
if t is str:
if columns[i].find('date_time') > -1:
c[i] += ":`timestamp$()"
else:
c[i] += ":`symbol$()"
elif t is float:
c[i] += ":`float$()"
elif t is int:
c[i] += ":`long$()"
keys = []
for i in primary_key_index:
keys.append(c[i])
for i in sorted(primary_key_index, reverse=True):
del c[i]
if len(keys) > 0:
command = '%s:([%s] %s)' % (table, '; '.join(keys), '; '.join(c))
else:
command = '%s:(%s)' % (table, '; '.join(c))
self.lock.acquire()
try:
self.conn.sync(command)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in creat statement(%s).\n%s" % (command, e))
finally:
self.lock.release()
return True