:return List of threads """ instmt.set_l2_depth(L2Depth(5)) instmt.set_prev_l2_depth(L2Depth(5)) instmt.set_instmt_snapshot_table_name( self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), instmt.get_instmt_name())) self.init_instmt_snapshot_table(instmt) instmt.set_recovered(False) t1 = threading.Thread( target=partial(self.get_order_book_worker, instmt)) t2 = threading.Thread(target=partial(self.get_trades_worker, instmt)) t1.start() t2.start() return [t1, t2] if __name__ == '__main__': Logger.init_log() exchange_name = 'Poloniex' instmt_name = 'BTCETH' instmt_code = 'BTC_ETH' instmt = Instrument(exchange_name, instmt_name, instmt_code) db_client = SqlClientTemplate() exch = ExchGwPoloniex([db_client]) instmt.set_l2_depth(L2Depth(5)) instmt.set_prev_l2_depth(L2Depth(5)) instmt.set_recovered(False) # exch.get_order_book_worker(instmt) exch.get_trades_worker(instmt)
raise Exception("Unknown type (%s) in kdb client select statement.\n%s" % (type(select_ret), select_ret)) return ret def delete(self, table, condition='1==1'): """ Delete rows from the table :param table: Table name :param condition: Where condition """ if condition == '1==1': statement = 'delete from `%s' % (table) else: statement = 'delete from `%s where %s' % (table, self.parse_condition(condition)) self.conn.sync(statement) return True if __name__ == '__main__': Logger.init_log() db_client = KdbPlusClient() db_client.connect(host='localhost', port=5000) db_client.create('test', ['c1', 'c2', 'c3', 'c4'], ['varchar(20)', 'int', 'decimal(8, 20)', 'int'], [0], False) db_client.insert('test', ['c1', 'c2', 'c3', 'c4'], ['abc', 1, 1.1, 5]) db_client.insert('test', ['c1', 'c2', 'c3', 'c4'], ['efg', 2, 2.2, 6]) db_client.insert('test', ['c1', 'c2', 'c3', 'c4'], ['hij', 3, 3.3, 7]) # Logger.info('test', db_client.select('test', columns=['*'])) Logger.info('test', db_client.select('test', columns=['c2', 'c3'], condition='c1 >= "abc" and c2 > 1')) # Logger.info('test', db_client.select('test', columns=['*'], orderby='c1 desc', limit=1)) db_client.delete('test', 'c1="abc"')
def main(): parser = argparse.ArgumentParser( description='Bitcoin exchange market data feed handler.') parser.add_argument('-instmts', action='store', help='Instrument subscription file.', default='subscriptions.ini') parser.add_argument('-exchtime', action='store_true', help='Use exchange timestamp.') parser.add_argument('-kdb', action='store_true', help='Use Kdb+ as database.') parser.add_argument('-csv', action='store_true', help='Use csv file as database.') parser.add_argument('-sqlite', action='store_true', help='Use SQLite database.') parser.add_argument('-mysql', action='store_true', help='Use MySQL.') parser.add_argument('-zmq', action='store_true', help='Use zmq publisher.') parser.add_argument('-kafka', action='store_true', help='Use kafka publisher.') parser.add_argument( '-mysqldest', action='store', dest='mysqldest', help='MySQL destination. Formatted as <name:pwd@host:port>', default='') parser.add_argument('-mysqlschema', action='store', dest='mysqlschema', help='MySQL schema.', default='') parser.add_argument('-kdbdest', action='store', dest='kdbdest', help='Kdb+ destination. Formatted as <host:port>', default='') parser.add_argument( '-zmqdest', action='store', dest='zmqdest', help='Zmq destination. For example \"tcp://127.0.0.1:3306\"', default='') parser.add_argument( '-kafkadest', action='store', dest='kafkadest', help='Kafka destination. For example \"127.0.0.1:9092\"', default='') parser.add_argument('-sqlitepath', action='store', dest='sqlitepath', help='SQLite database path', default='') parser.add_argument('-csvpath', action='store', dest='csvpath', help='Csv file path', default='') parser.add_argument('-output', action='store', dest='output', help='Verbose output file path') args = parser.parse_args() Logger.init_log(args.output) db_clients = [] is_database_defined = False if args.sqlite: db_client = SqliteClient() db_client.connect(path=args.sqlitepath) db_clients.append(db_client) is_database_defined = True if args.mysql: db_client = MysqlClient() mysqldest = args.mysqldest logon_credential = mysqldest.split('@')[0] connection = mysqldest.split('@')[1] db_client.connect(host=connection.split(':')[0], port=int(connection.split(':')[1]), user=logon_credential.split(':')[0], pwd=logon_credential.split(':')[1], schema=args.mysqlschema) db_clients.append(db_client) is_database_defined = True if args.csv: if args.csvpath != '': db_client = FileClient(dir=args.csvpath) else: db_client = FileClient() db_clients.append(db_client) is_database_defined = True if args.kdb: db_client = KdbPlusClient() db_client.connect(host=args.kdbdest.split(':')[0], port=int(args.kdbdest.split(':')[1])) db_clients.append(db_client) is_database_defined = True if args.zmq: db_client = ZmqClient() db_client.connect(addr=args.zmqdest) db_clients.append(db_client) is_database_defined = True if args.kafka: db_client = KafkaClient() db_client.connect(addr=args.kafkadest) db_clients.append(db_client) is_database_defined = True if not is_database_defined: print('Error: Please define which database is used.') parser.print_help() sys.exit(1) # Subscription instruments if args.instmts is None or len(args.instmts) == 0: print( 'Error: Please define the instrument subscription list. You can refer to subscriptions.ini.' ) parser.print_help() sys.exit(1) # Use exchange timestamp rather than local timestamp if args.exchtime: ExchangeGateway.is_local_timestamp = False # Initialize subscriptions subscription_instmts = SubscriptionManager( args.instmts).get_subscriptions() if len(subscription_instmts) == 0: print( 'Error: No instrument is found in the subscription file. ' + 'Please check the file path and the content of the subscription file.' ) parser.print_help() sys.exit(1) # Initialize snapshot destination ExchangeGateway.init_snapshot_table(db_clients) Logger.info('[main]', 'Subscription file = %s' % args.instmts) log_str = 'Exchange/Instrument/InstrumentCode:\n' for instmt in subscription_instmts: log_str += '%s/%s/%s\n' % (instmt.exchange_name, instmt.instmt_name, instmt.instmt_code) Logger.info('[main]', log_str) exch_gws = [] exch_gws.append(ExchGwBtccSpot(db_clients)) exch_gws.append(ExchGwBtccFuture(db_clients)) exch_gws.append(ExchGwBitmex(db_clients)) exch_gws.append(ExchGwBitfinex(db_clients)) exch_gws.append(ExchGwOkCoin(db_clients)) exch_gws.append(ExchGwKraken(db_clients)) exch_gws.append(ExchGwGdax(db_clients)) exch_gws.append(ExchGwBitstamp(db_clients)) exch_gws.append(ExchGwGatecoin(db_clients)) exch_gws.append(ExchGwQuoine(db_clients)) exch_gws.append(ExchGwPoloniex(db_clients)) exch_gws.append(ExchGwBittrex(db_clients)) exch_gws.append(ExchGwYunbi(db_clients)) exch_gws.append(ExchGwLiqui(db_clients)) exch_gws.append(ExchGwBinance(db_clients)) exch_gws.append(ExchGwCryptopia(db_clients)) threads = [] for exch in exch_gws: for instmt in subscription_instmts: if instmt.get_exchange_name() == exch.get_exchange_name(): Logger.info("[main]", "Starting instrument %s-%s..." % \ (instmt.get_exchange_name(), instmt.get_instmt_name())) threads += exch.start(instmt)
def main(): parser = argparse.ArgumentParser(description='Bitcoin exchange market data feed handler.') parser.add_argument('-instmts', action='store', help='Instrument subscription file.', default='subscriptions.ini') parser.add_argument('-exchtime', action='store_true', help='Use exchange timestamp.') parser.add_argument('-kdb', action='store_true', help='Use Kdb+ as database.') parser.add_argument('-csv', action='store_true', help='Use csv file as database.') parser.add_argument('-sqlite', action='store_true', help='Use SQLite database.') parser.add_argument('-mysql', action='store_true', help='Use MySQL.') parser.add_argument('-zmq', action='store_true', help='Use zmq publisher.') parser.add_argument('-mysqldest', action='store', dest='mysqldest', help='MySQL destination. Formatted as <name:pwd@host:port>', default='') parser.add_argument('-mysqlschema', action='store', dest='mysqlschema', help='MySQL schema.', default='') parser.add_argument('-kdbdest', action='store', dest='kdbdest', help='Kdb+ destination. Formatted as <host:port>', default='') parser.add_argument('-zmqdest', action='store', dest='zmqdest', help='Zmq destination. For example \"tcp://127.0.0.1:3306\"', default='') parser.add_argument('-sqlitepath', action='store', dest='sqlitepath', help='SQLite database path', default='') parser.add_argument('-csvpath', action='store', dest='csvpath', help='Csv file path', default='') parser.add_argument('-output', action='store', dest='output', help='Verbose output file path') args = parser.parse_args() Logger.init_log(args.output) db_clients = [] is_database_defined = False if args.sqlite: db_client = SqliteClient() db_client.connect(path=args.sqlitepath) db_clients.append(db_client) is_database_defined = True if args.mysql: db_client = MysqlClient() mysqldest = args.mysqldest logon_credential = mysqldest.split('@')[0] connection = mysqldest.split('@')[1] db_client.connect(host=connection.split(':')[0], port=int(connection.split(':')[1]), user=logon_credential.split(':')[0], pwd=logon_credential.split(':')[1], schema=args.mysqlschema) db_clients.append(db_client) is_database_defined = True if args.csv: if args.csvpath != '': db_client = FileClient(dir=args.csvpath) else: db_client = FileClient() db_clients.append(db_client) is_database_defined = True if args.kdb: db_client = KdbPlusClient() db_client.connect(host=args.kdbdest.split(':')[0], port=int(args.kdbdest.split(':')[1])) db_clients.append(db_client) is_database_defined = True if args.zmq: db_client = ZmqClient() db_client.connect(addr=args.zmqdest) db_clients.append(db_client) is_database_defined = True if not is_database_defined: print('Error: Please define which database is used.') parser.print_help() sys.exit(1) # Subscription instruments if args.instmts is None or len(args.instmts) == 0: print('Error: Please define the instrument subscription list. You can refer to subscriptions.ini.') parser.print_help() sys.exit(1) # Use exchange timestamp rather than local timestamp if args.exchtime: ExchangeGateway.is_local_timestamp = False subscription_instmts = SubscriptionManager(args.instmts).get_subscriptions() ExchangeGateway.init_snapshot_table(db_clients) Logger.info('[main]', 'Subscription file = %s' % args.instmts) log_str = 'Exchange/Instrument/InstrumentCode:\n' for instmt in subscription_instmts: log_str += '%s/%s/%s\n' % (instmt.exchange_name, instmt.instmt_name, instmt.instmt_code) Logger.info('[main]', log_str) exch_gws = [] exch_gws.append(ExchGwBtccSpot(db_clients)) exch_gws.append(ExchGwBtccFuture(db_clients)) exch_gws.append(ExchGwBitmex(db_clients)) exch_gws.append(ExchGwBitfinex(db_clients)) exch_gws.append(ExchGwOkCoin(db_clients)) exch_gws.append(ExchGwKraken(db_clients)) exch_gws.append(ExchGwGdax(db_clients)) exch_gws.append(ExchGwBitstamp(db_clients)) exch_gws.append(ExchGwGatecoin(db_clients)) exch_gws.append(ExchGwQuoine(db_clients)) exch_gws.append(ExchGwPoloniex(db_clients)) exch_gws.append(ExchGwBittrex(db_clients)) threads = [] for exch in exch_gws: for instmt in subscription_instmts: if instmt.get_exchange_name() == exch.get_exchange_name(): Logger.info("[main]", "Starting instrument %s-%s..." % \ (instmt.get_exchange_name(), instmt.get_instmt_name())) threads += exch.start(instmt)