def on_message_handler(self, instmt, message): """ Incoming message handler :param instmt: Instrument :param message: Message """ if 'ping' in message: #handle ping response ts = message['ping'] self.api_socket.send(json.dumps({'pong': ts})) elif 'ch' in message: if 'trade.detail' in message['ch']: trades = self.api_socket.parse_trade(instmt, message['tick']['data']) for trade in trades: if trade.trade_id != instmt.get_exch_trade_id(): instmt.incr_trade_id() instmt.set_exch_trade_id(trade.trade_id) self.insert_trade(instmt, trade) elif 'depth.step' in message['ch']: instmt.set_prev_l2_depth(instmt.get_l2_depth().copy()) self.api_socket.parse_l2_depth(instmt, message['tick']) if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): instmt.incr_order_book_id() self.insert_order_book(instmt) else: Logger.error(self.__class__.__name__, 'Not Trade or Market')
def insert_data_worker(self): while True: time.sleep(0.5) if not self.q.empty(): items = [] i = 0 while not self.q.empty(): items.append(self.q.get()) self.q.task_done() i += 1 self.lock.acquire() try: rv = self.client.write_points(items, database=self.dbname, time_precision='u') if not rv: Logger.error(self.__class__.__name__, "Error writing to InfluxDb") except Exception as e: Logger.error(self.__class__.__name__, "InfluxDb error: %s" % (e)) self.lock.release() else: pass
def on_message_handler(self, instmt, message): """ Incoming message handler :param instmt: Instrument :param message: Message """ keys = message.keys() if 'info' in keys: Logger.info(self.__class__.__name__, message['info']) elif 'subscribe' in keys: Logger.info(self.__class__.__name__, 'Subscription of %s is %s' % \ (message['request']['args'], \ 'successful' if message['success'] else 'failed')) elif 'table' in keys: if message['table'] == 'trade': for trade_raw in message['data']: if trade_raw["symbol"] == instmt.get_instmt_code(): # Filter out the initial subscriptions trade = self.api_socket.parse_trade(instmt, trade_raw) if trade.trade_id != instmt.get_exch_trade_id(): instmt.incr_trade_id() instmt.set_exch_trade_id(trade.trade_id) self.insert_trade(instmt, trade) elif message['table'] == 'orderBookL2': l2_depth = self.api_socket.parse_l2_depth(instmt, message) if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): instmt.set_prev_l2_depth(instmt.get_l2_depth()) instmt.set_l2_depth(l2_depth) instmt.incr_order_book_id() self.insert_order_book(instmt) else: Logger.info(self.__class__.__name__, json.dumps(message,indent=2)) else: Logger.error(self.__class__.__name__, "Unrecognised message:\n" + json.dumps(message))
def on_message_handler(self, instmt, message): """ Incoming message handler :param instmt: Instrument :param message: Message """ keys = message.keys() if 'info' in keys: Logger.info(self.__class__.__name__, message['info']) elif 'subscribe' in keys: Logger.info(self.__class__.__name__, 'Subscription of %s is %s' % \ (message['request']['args'], \ 'successful' if message['success'] else 'failed')) elif 'table' in keys: if message['table'] == 'trade': for trade_raw in message['data']: if trade_raw["symbol"] == instmt.get_instmt_code(): # Filter out the initial subscriptions trade = self.api_socket.parse_trade(instmt, trade_raw) if trade.trade_id != instmt.get_exch_trade_id(): instmt.incr_trade_id() instmt.set_exch_trade_id(trade.trade_id) self.insert_trade(instmt, trade) elif message['table'] == 'orderBook10': for data in message['data']: if data["symbol"] == instmt.get_instmt_code(): instmt.set_prev_l2_depth(instmt.get_l2_depth().copy()) self.api_socket.parse_l2_depth(instmt, data) if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): instmt.incr_order_book_id() self.insert_order_book(instmt) else: Logger.info(self.__class__.__name__, json.dumps(message,indent=2)) else: Logger.error(self.__class__.__name__, "Unrecognised message:\n" + json.dumps(message))
def get_order_book_worker(self, instmt): """ Get order book worker :param instmt: Instrument """ while True: ExchGwQuoine.last_query_time_lock.acquire() if datetime.now() - ExchGwQuoine.last_query_time < timedelta( seconds=ExchGwQuoine.waiting_seconds): ExchGwQuoine.last_query_time_lock.release() time.sleep(random.uniform(0, 1)) else: ExchGwQuoine.last_query_time = datetime.now() try: l2_depth = self.api_socket.get_order_book(instmt) if l2_depth is not None and l2_depth.is_diff( instmt.get_l2_depth()): instmt.set_prev_l2_depth(instmt.get_l2_depth()) instmt.set_l2_depth(l2_depth) instmt.incr_order_book_id() self.insert_order_book(instmt) except Exception as e: Logger.error(self.__class__.__name__, "Error in order book: %s" % e) ExchGwQuoine.last_query_time_lock.release()
def get_trades_worker(self, instmt): """ Get order book worker thread :param instmt: Instrument name """ while True: try: ret = self.api_socket.get_trades(instmt) if ret is None or len(ret) == 0: time.sleep(1) continue except Exception as e: Logger.error(self.__class__.__name__, "Error in trades: %s" % e) for trade in ret: assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id) assert isinstance(instmt.get_exch_trade_id(), str), \ "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) if int(trade.trade_id) > int(instmt.get_exch_trade_id()): instmt.set_exch_trade_id(trade.trade_id) instmt.incr_trade_id() self.insert_trade(instmt, trade) # After the first time of getting the trade, indicate the instrument # is recovered if not instmt.get_recovered(): instmt.set_recovered(True) time.sleep(1)
def send_order(amount, source, symbol, _type, price=0): """ :param amount: :param source: :param symbol: :param _type: 可选值 {buy-market:市价买, sell-market:市价卖, buy-limit:限价买, sell-limit:限价卖} :param price: :return: """ try: accounts = get_accounts() acct_id = accounts['data'][0]['id'] except BaseException as e: #xie begin , #print 'get acct_id error.%s'%e Logger.error("HuobiService", "Unrecognised message:\n" % e) #xie end acct_id = hbu.ACCOUNT_ID params = { "account-id": acct_id, "amount": amount, "symbol": symbol, "type": _type, "source": source } if price: params["price"] = price url = '/v1/order/orders/place' return hbu.api_key_post(params, url)
def get_trades_worker(self, instmt): """ Get order book worker thread :param instmt: Instrument name """ while True: try: ret = self.api_socket.get_trades(instmt) if ret is None or len(ret) == 0: time.sleep(1) continue except Exception as e: Logger.error(self.__class__.__name__, "Error in trades: %s" % e) time.sleep(1) continue for trade in ret: assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type( trade.trade_id), trade.trade_id) assert isinstance(instmt.get_exch_trade_id(), str), \ "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) if int(trade.trade_id) > int(instmt.get_exch_trade_id()): instmt.set_exch_trade_id(trade.trade_id) instmt.incr_trade_id() self.insert_trade(instmt, trade) # After the first time of getting the trade, indicate the instrument # is recovered if not instmt.get_recovered(): instmt.set_recovered(True) time.sleep(1)
def create(self, table, columns, types, primary_key_index=[], is_ifnotexists=True): """ Create table in the database. Caveat - Assign the first few column as the keys!!! :param table: Table name :param columns: Column array :param types: Type array :param is_ifnotexists: Create table if not exists keyword """ if len(columns) != len(types): raise Exception("Incorrect create statement. Number of columns and that of types are different.\n%s\n%s" % \ (columns, types)) if is_ifnotexists: ret = self.conn("\\v") if ret is not None: for t in ret: if table == self.decode_qtypes(t): Logger.info(self.__class__.__name__, "Table %s has been created." % table) return True Logger.info(self.__class__.__name__, "Table %s is going to be created." % table) c = columns[:] for i in range(0, len(types)): t = self.convert_type(types[i]) if t is str: if columns[i].find('date_time') > -1: c[i] += ":`timestamp$()" else: c[i] += ":`symbol$()" elif t is float: c[i] += ":`float$()" elif t is int: c[i] += ":`long$()" keys = [] for i in primary_key_index: keys.append(c[i]) for i in sorted(primary_key_index, reverse=True): del c[i] if len(keys) > 0: command = '%s:([%s] %s)' % (table, '; '.join(keys), '; '.join(c)) else: command = '%s:(%s)' % (table, '; '.join(c)) self.lock.acquire() try: self.conn.sync(command) except Exception as e: Logger.error(self.__class__.__name__, "Error in creat statement(%s).\n%s" % (command, e)) finally: self.lock.release() return True
def insert(self, table, columns, types, values, primary_key_index=[], is_orreplace=False, is_commit=True): """ Insert into the table :param table: Table name :param columns: Column array :param types: Type array :param values: Value array :param primary_key_index: An array of indices of primary keys in columns, e.g. [0] means the first column is the primary key :param is_orreplace: Indicate if the query is "INSERT OR REPLACE" """ if is_orreplace: # skip snaphosts return True exchange_name, tick = self.split_table_name(table) if len(columns) != len(values): return False idata = {"measurement": tick, "tags": {"exchange": exchange_name}} fields = {} for i in range(0, len(columns)): if columns[i] in self.column_time_name: idata["time"] = int(self.time_to_unix(values[i])) # store only trades (skip orders) if self.column_update_type in columns[i]: if int(values[i]) is not 2: return True if columns[i] in self.column_ignore: continue else: if "int" in types[i]: fields[columns[i]] = int(values[i]) elif "decimal" in types[i]: fields[columns[i]] = float(values[i]) elif "varchar" in types[i]: fields[columns[i]] = values[i] else: Logger.error(self.__class__.__name__, "Unkown value type:{}".format(types[i])) return True idata["fields"] = fields self.q.put(idata) return True
def on_message_handler(self, instmt, message): """ Incoming message handler :param instmt: Instrument :param message: Message """ if not message: return keys = message.keys() if "bids" in keys: self.order_book = self.api_socket.parse_l2_depth(instmt, message) # Insert only if the first 5 levels are different if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): instmt.incr_order_book_id() self.insert_order_book(instmt) elif "create_update" in keys: if message['create_update']: message['create_update'].update( {"timestamp": message['timestamp']}) self.api_socket.parse_l2_depth(instmt, message['create_update']) # Insert only if the first 5 levels are different if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): instmt.incr_order_book_id() self.insert_order_book(instmt) elif message['delete_update']: message['delete_update'].update( {"timestamp": message['timestamp']}) self.api_socket.parse_l2_depth(instmt, message['delete_update']) # Insert only if the first 5 levels are different if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): instmt.incr_order_book_id() self.insert_order_book(instmt) elif message['trade_updates']: for new_trade in message['trade_updates']: new_trade.update({"timestamp": message['timestamp']}) trade = self.api_socket.parse_trade(instmt, new_trade) self.api_socket.parse_l2_depth(instmt, new_trade) if trade.trade_id != instmt.get_exch_trade_id(): instmt.incr_trade_id() instmt.set_exch_trade_id(trade.trade_id) self.insert_trade(instmt, trade) else: Logger.error(self.__class__.__name__, "Unrecognised message:\n" + json.dumps(message))
def get_order_book(cls, instmt): """ Get order book :param instmt: Instrument :return: Object L2Depth """ res = cls.request(cls.get_order_book_link(instmt)) if len(res) > 0 and 'error' in res and len(res['error']) == 0: res = list(res['result'].values())[0] return cls.parse_l2_depth(instmt=instmt, raw=res) else: Logger.error(cls.__name__, "Cannot parse the order book. Return:\n%s" % res) return None
def get_order_book_worker(self, instmt): """ Get order book worker :param instmt: Instrument """ while True: try: l2_depth = self.api_socket.get_order_book(instmt) if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): instmt.set_prev_l2_depth(instmt.get_l2_depth()) instmt.set_l2_depth(l2_depth) instmt.incr_order_book_id() self.insert_order_book(instmt) except Exception as e: Logger.error(self.__class__.__name__, "Error in order book: %s" % e) time.sleep(1)
def get_order_book_worker(self, instmt): """ Get order book worker :param instmt: Instrument """ while True: try: l2_depth = self.api_socket2.get_order_book(instmt) if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): instmt.set_prev_l2_depth(instmt.get_l2_depth()) instmt.set_l2_depth(l2_depth) instmt.incr_order_book_id() self.insert_order_book(instmt) except Exception as e: Logger.error(self.__class__.__name__, "Error in order book: %s" % e) time.sleep(1)
def insert(self, table, columns, types, values, primary_key_index=[], is_orreplace=False, is_commit=True): """ Insert into the table :param table: Table name :param columns: Column array :param types: Type array :param values: Value array :param primary_key_index: An array of indices of primary keys in columns, e.g. [0] means the first column is the primary key :param is_orreplace: Indicate if the query is "INSERT OR REPLACE" """ ret = dict(zip(columns, values)) ret['table'] = table self.lock.acquire() # print(ret) # print('columns:', columns) # print('values:', values) future = self.conn.send(table, value=ret) result = True # Block for 'synchronous' sends try: record_metadata = future.get(timeout=60) # print(record_metadata) Logger.info( self.__class__.__name__, "topic: %s, offset: %s" % (record_metadata.topic, record_metadata.offset)) except Exception as ex: Logger.error(self.__class__.__name__, "exception in producer:%s" % ex) # traceback.print_exc() result = False # raise Exception("kafka send failed.") finally: self.lock.release() return result
def get_balance(acct_id=None): """ :param acct_id :return: """ if not acct_id: try: accounts = get_accounts() acct_id = ACCOUNT_ID = accounts['data'][0]['id'] except BaseException as e: #xie begin ,print 'get acct_id error.%s'%e Logger.error("HuobiService", "Unrecognised message:\n" % e) #xie end acct_id = ACCOUNT_ID url = "/v1/account/accounts/{0}/balance".format(acct_id) params = {"account-id": acct_id} return hbu.api_key_get(params, url)
def get_trades_worker(self, instmt): """ Get order book worker thread :param instmt: Instrument name """ while True: ExchGwQuoine.last_query_time_lock.acquire() if datetime.now() - ExchGwQuoine.last_query_time < timedelta( seconds=ExchGwQuoine.waiting_seconds): ExchGwQuoine.last_query_time_lock.release() time.sleep(random.uniform(0, 1)) else: ExchGwQuoine.last_query_time = datetime.now() try: ret = self.api_socket.get_trades(instmt) if ret is None or len(ret) == 0: ExchGwQuoine.last_query_time_lock.release() continue for trade in ret: assert isinstance( trade.trade_id, str), "trade.trade_id(%s) = %s" % (type( trade.trade_id), trade.trade_id) assert isinstance(instmt.get_exch_trade_id(), str), \ "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) if int(trade.trade_id) > int( instmt.get_exch_trade_id()): instmt.set_exch_trade_id(trade.trade_id) instmt.incr_trade_id() self.insert_trade(instmt, trade) # After the first time of getting the trade, indicate the instrument # is recovered if not instmt.get_recovered(): instmt.set_recovered(True) except Exception as e: Logger.error(self.__class__.__name__, "Error in trades: %s" % e) ExchGwQuoine.last_query_time_lock.release()
def get_order_book_worker(self, instmt): """ Get order book worker :param instmt: Instrument """ while True: ExchGwQuoine.last_query_time_lock.acquire() if datetime.now() - ExchGwQuoine.last_query_time < timedelta(seconds=ExchGwQuoine.waiting_seconds): ExchGwQuoine.last_query_time_lock.release() time.sleep(random.uniform(0, 1)) else: ExchGwQuoine.last_query_time = datetime.now() try: l2_depth = self.api_socket.get_order_book(instmt) if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): instmt.set_prev_l2_depth(instmt.get_l2_depth()) instmt.set_l2_depth(l2_depth) instmt.incr_order_book_id() self.insert_order_book(instmt) except Exception as e: Logger.error(self.__class__.__name__, "Error in order book: %s" % e) ExchGwQuoine.last_query_time_lock.release()
def get_trades_worker(self, instmt): """ Get order book worker thread :param instmt: Instrument name """ instmt.set_recovered(False) while True: try: ret = self.api_socket.get_trades(instmt) for trade in ret: instmt.incr_trade_id() self.insert_trade(instmt, trade) # After the first time of getting the trade, indicate the instrument # is recovered if not instmt.get_recovered(): instmt.set_recovered(True) except Exception as e: Logger.error(self.__class__.__name__, "Error in trades: %s\nReturn: %s" % (e, ret)) time.sleep(0.5)
def get_trades_worker(self, instmt): """ Get order book worker thread :param instmt: Instrument name """ while True: ExchGwQuoine.last_query_time_lock.acquire() if datetime.now() - ExchGwQuoine.last_query_time < timedelta(seconds=ExchGwQuoine.waiting_seconds): ExchGwQuoine.last_query_time_lock.release() time.sleep(random.uniform(0, 1)) else: ExchGwQuoine.last_query_time = datetime.now() try: ret = self.api_socket.get_trades(instmt) if ret is None or len(ret) == 0: ExchGwQuoine.last_query_time_lock.release() continue for trade in ret: assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id) assert isinstance(instmt.get_exch_trade_id(), str), \ "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) if int(trade.trade_id) > int(instmt.get_exch_trade_id()): instmt.set_exch_trade_id(trade.trade_id) instmt.incr_trade_id() self.insert_trade(instmt, trade) # After the first time of getting the trade, indicate the instrument # is recovered if not instmt.get_recovered(): instmt.set_recovered(True) except Exception as e: Logger.error(self.__class__.__name__, "Error in trades: %s" % e) ExchGwQuoine.last_query_time_lock.release()
def create(self, table, columns, types, primary_key_index=(), is_ifnotexists=True): """ Create table in the database. Caveat - Assign the first few column as the keys!!! :param table: Table name :param columns: Column array :param types: Type array :param is_ifnotexists: Create table if not exists keyword """ if len(columns) != len(types): raise Exception("Incorrect create statement. Number of columns and that of types are different.\n%s\n%s" % \ (columns, types)) if is_ifnotexists: ret = self.conn("\\v") if ret is not None: for t in ret: if table == self.decode_qtypes(t): Logger.info(self.__class__.__name__, "Table %s has been created." % table) return True Logger.info(self.__class__.__name__, "Table %s is going to be created." % table) c = columns[:] for i in range(0, len(types)): t = self.convert_type(types[i]) if t is str: if columns[i].find('date_time') > -1: c[i] += ":`timestamp$()" else: c[i] += ":`symbol$()" elif t is float: c[i] += ":`float$()" elif t is int: c[i] += ":`long$()" keys = [] for i in primary_key_index: keys.append(c[i]) for i in sorted(primary_key_index, reverse=True): del c[i] if len(keys) > 0: command = '%s:([%s] %s)' % (table, '; '.join(keys), '; '.join(c)) else: command = '%s:(%s)' % (table, '; '.join(c)) self.lock.acquire() try: self.conn.sync(command) except Exception as e: Logger.error(self.__class__.__name__, "Error in creat statement(%s).\n%s" % (command, e)) finally: self.lock.release() return True
def __on_error(self, ws, error): Logger.error(self.__class__.__name__, "Socket <%s> error:\n %s" % (self.id, error)) if len(self.on_error_handlers) > 0: for handler in self.on_error_handlers: handler(ws, error)