:return List of threads
"""
instmt.set_l2_depth(L2Depth(5))
instmt.set_prev_l2_depth(L2Depth(5))
instmt.set_instmt_snapshot_table_name(
self.get_instmt_snapshot_table_name(instmt.get_exchange_name(),
instmt.get_instmt_name()))
self.init_instmt_snapshot_table(instmt)
instmt.set_recovered(False)
t1 = threading.Thread(
target=partial(self.get_order_book_worker, instmt))
t2 = threading.Thread(target=partial(self.get_trades_worker, instmt))
t1.start()
t2.start()
return [t1, t2]
if __name__ == '__main__':
Logger.init_log()
exchange_name = 'Poloniex'
instmt_name = 'BTCETH'
instmt_code = 'BTC_ETH'
instmt = Instrument(exchange_name, instmt_name, instmt_code)
db_client = SqlClientTemplate()
exch = ExchGwPoloniex([db_client])
instmt.set_l2_depth(L2Depth(5))
instmt.set_prev_l2_depth(L2Depth(5))
instmt.set_recovered(False)
# exch.get_order_book_worker(instmt)
exch.get_trades_worker(instmt)
raise Exception("Unknown type (%s) in kdb client select statement.\n%s" % (type(select_ret), select_ret))
return ret
def delete(self, table, condition='1==1'):
"""
Delete rows from the table
:param table: Table name
:param condition: Where condition
"""
if condition == '1==1':
statement = 'delete from `%s' % (table)
else:
statement = 'delete from `%s where %s' % (table, self.parse_condition(condition))
self.conn.sync(statement)
return True
if __name__ == '__main__':
Logger.init_log()
db_client = KdbPlusClient()
db_client.connect(host='localhost', port=5000)
db_client.create('test', ['c1', 'c2', 'c3', 'c4'], ['varchar(20)', 'int', 'decimal(8, 20)', 'int'], [0], False)
db_client.insert('test', ['c1', 'c2', 'c3', 'c4'], ['abc', 1, 1.1, 5])
db_client.insert('test', ['c1', 'c2', 'c3', 'c4'], ['efg', 2, 2.2, 6])
db_client.insert('test', ['c1', 'c2', 'c3', 'c4'], ['hij', 3, 3.3, 7])
# Logger.info('test', db_client.select('test', columns=['*']))
Logger.info('test', db_client.select('test', columns=['c2', 'c3'], condition='c1 >= "abc" and c2 > 1'))
# Logger.info('test', db_client.select('test', columns=['*'], orderby='c1 desc', limit=1))
db_client.delete('test', 'c1="abc"')
def main():
parser = argparse.ArgumentParser(
description='Bitcoin exchange market data feed handler.')
parser.add_argument('-instmts',
action='store',
help='Instrument subscription file.',
default='subscriptions.ini')
parser.add_argument('-exchtime',
action='store_true',
help='Use exchange timestamp.')
parser.add_argument('-kdb',
action='store_true',
help='Use Kdb+ as database.')
parser.add_argument('-csv',
action='store_true',
help='Use csv file as database.')
parser.add_argument('-sqlite',
action='store_true',
help='Use SQLite database.')
parser.add_argument('-mysql', action='store_true', help='Use MySQL.')
parser.add_argument('-zmq', action='store_true', help='Use zmq publisher.')
parser.add_argument('-kafka',
action='store_true',
help='Use kafka publisher.')
parser.add_argument(
'-mysqldest',
action='store',
dest='mysqldest',
help='MySQL destination. Formatted as <name:pwd@host:port>',
default='')
parser.add_argument('-mysqlschema',
action='store',
dest='mysqlschema',
help='MySQL schema.',
default='')
parser.add_argument('-kdbdest',
action='store',
dest='kdbdest',
help='Kdb+ destination. Formatted as <host:port>',
default='')
parser.add_argument(
'-zmqdest',
action='store',
dest='zmqdest',
help='Zmq destination. For example \"tcp://127.0.0.1:3306\"',
default='')
parser.add_argument(
'-kafkadest',
action='store',
dest='kafkadest',
help='Kafka destination. For example \"127.0.0.1:9092\"',
default='')
parser.add_argument('-sqlitepath',
action='store',
dest='sqlitepath',
help='SQLite database path',
default='')
parser.add_argument('-csvpath',
action='store',
dest='csvpath',
help='Csv file path',
default='')
parser.add_argument('-output',
action='store',
dest='output',
help='Verbose output file path')
args = parser.parse_args()
Logger.init_log(args.output)
db_clients = []
is_database_defined = False
if args.sqlite:
db_client = SqliteClient()
db_client.connect(path=args.sqlitepath)
db_clients.append(db_client)
is_database_defined = True
if args.mysql:
db_client = MysqlClient()
mysqldest = args.mysqldest
logon_credential = mysqldest.split('@')[0]
connection = mysqldest.split('@')[1]
db_client.connect(host=connection.split(':')[0],
port=int(connection.split(':')[1]),
user=logon_credential.split(':')[0],
pwd=logon_credential.split(':')[1],
schema=args.mysqlschema)
db_clients.append(db_client)
is_database_defined = True
if args.csv:
if args.csvpath != '':
db_client = FileClient(dir=args.csvpath)
else:
db_client = FileClient()
db_clients.append(db_client)
is_database_defined = True
if args.kdb:
db_client = KdbPlusClient()
db_client.connect(host=args.kdbdest.split(':')[0],
port=int(args.kdbdest.split(':')[1]))
db_clients.append(db_client)
is_database_defined = True
if args.zmq:
db_client = ZmqClient()
db_client.connect(addr=args.zmqdest)
db_clients.append(db_client)
is_database_defined = True
if args.kafka:
db_client = KafkaClient()
db_client.connect(addr=args.kafkadest)
db_clients.append(db_client)
is_database_defined = True
if not is_database_defined:
print('Error: Please define which database is used.')
parser.print_help()
sys.exit(1)
# Subscription instruments
if args.instmts is None or len(args.instmts) == 0:
print(
'Error: Please define the instrument subscription list. You can refer to subscriptions.ini.'
)
parser.print_help()
sys.exit(1)
# Use exchange timestamp rather than local timestamp
if args.exchtime:
ExchangeGateway.is_local_timestamp = False
# Initialize subscriptions
subscription_instmts = SubscriptionManager(
args.instmts).get_subscriptions()
if len(subscription_instmts) == 0:
print(
'Error: No instrument is found in the subscription file. ' +
'Please check the file path and the content of the subscription file.'
)
parser.print_help()
sys.exit(1)
# Initialize snapshot destination
ExchangeGateway.init_snapshot_table(db_clients)
Logger.info('[main]', 'Subscription file = %s' % args.instmts)
log_str = 'Exchange/Instrument/InstrumentCode:\n'
for instmt in subscription_instmts:
log_str += '%s/%s/%s\n' % (instmt.exchange_name, instmt.instmt_name,
instmt.instmt_code)
Logger.info('[main]', log_str)
exch_gws = []
exch_gws.append(ExchGwBtccSpot(db_clients))
exch_gws.append(ExchGwBtccFuture(db_clients))
exch_gws.append(ExchGwBitmex(db_clients))
exch_gws.append(ExchGwBitfinex(db_clients))
exch_gws.append(ExchGwOkCoin(db_clients))
exch_gws.append(ExchGwKraken(db_clients))
exch_gws.append(ExchGwGdax(db_clients))
exch_gws.append(ExchGwBitstamp(db_clients))
exch_gws.append(ExchGwGatecoin(db_clients))
exch_gws.append(ExchGwQuoine(db_clients))
exch_gws.append(ExchGwPoloniex(db_clients))
exch_gws.append(ExchGwBittrex(db_clients))
exch_gws.append(ExchGwYunbi(db_clients))
exch_gws.append(ExchGwLiqui(db_clients))
exch_gws.append(ExchGwBinance(db_clients))
exch_gws.append(ExchGwCryptopia(db_clients))
threads = []
for exch in exch_gws:
for instmt in subscription_instmts:
if instmt.get_exchange_name() == exch.get_exchange_name():
Logger.info("[main]", "Starting instrument %s-%s..." % \
(instmt.get_exchange_name(), instmt.get_instmt_name()))
threads += exch.start(instmt)
def main():
parser = argparse.ArgumentParser(description='Bitcoin exchange market data feed handler.')
parser.add_argument('-instmts', action='store', help='Instrument subscription file.', default='subscriptions.ini')
parser.add_argument('-exchtime', action='store_true', help='Use exchange timestamp.')
parser.add_argument('-kdb', action='store_true', help='Use Kdb+ as database.')
parser.add_argument('-csv', action='store_true', help='Use csv file as database.')
parser.add_argument('-sqlite', action='store_true', help='Use SQLite database.')
parser.add_argument('-mysql', action='store_true', help='Use MySQL.')
parser.add_argument('-zmq', action='store_true', help='Use zmq publisher.')
parser.add_argument('-mysqldest', action='store', dest='mysqldest',
help='MySQL destination. Formatted as <name:pwd@host:port>',
default='')
parser.add_argument('-mysqlschema', action='store', dest='mysqlschema',
help='MySQL schema.',
default='')
parser.add_argument('-kdbdest', action='store', dest='kdbdest',
help='Kdb+ destination. Formatted as <host:port>',
default='')
parser.add_argument('-zmqdest', action='store', dest='zmqdest',
help='Zmq destination. For example \"tcp://127.0.0.1:3306\"',
default='')
parser.add_argument('-sqlitepath', action='store', dest='sqlitepath',
help='SQLite database path',
default='')
parser.add_argument('-csvpath', action='store', dest='csvpath',
help='Csv file path',
default='')
parser.add_argument('-output', action='store', dest='output',
help='Verbose output file path')
args = parser.parse_args()
Logger.init_log(args.output)
db_clients = []
is_database_defined = False
if args.sqlite:
db_client = SqliteClient()
db_client.connect(path=args.sqlitepath)
db_clients.append(db_client)
is_database_defined = True
if args.mysql:
db_client = MysqlClient()
mysqldest = args.mysqldest
logon_credential = mysqldest.split('@')[0]
connection = mysqldest.split('@')[1]
db_client.connect(host=connection.split(':')[0],
port=int(connection.split(':')[1]),
user=logon_credential.split(':')[0],
pwd=logon_credential.split(':')[1],
schema=args.mysqlschema)
db_clients.append(db_client)
is_database_defined = True
if args.csv:
if args.csvpath != '':
db_client = FileClient(dir=args.csvpath)
else:
db_client = FileClient()
db_clients.append(db_client)
is_database_defined = True
if args.kdb:
db_client = KdbPlusClient()
db_client.connect(host=args.kdbdest.split(':')[0], port=int(args.kdbdest.split(':')[1]))
db_clients.append(db_client)
is_database_defined = True
if args.zmq:
db_client = ZmqClient()
db_client.connect(addr=args.zmqdest)
db_clients.append(db_client)
is_database_defined = True
if not is_database_defined:
print('Error: Please define which database is used.')
parser.print_help()
sys.exit(1)
# Subscription instruments
if args.instmts is None or len(args.instmts) == 0:
print('Error: Please define the instrument subscription list. You can refer to subscriptions.ini.')
parser.print_help()
sys.exit(1)
# Use exchange timestamp rather than local timestamp
if args.exchtime:
ExchangeGateway.is_local_timestamp = False
subscription_instmts = SubscriptionManager(args.instmts).get_subscriptions()
ExchangeGateway.init_snapshot_table(db_clients)
Logger.info('[main]', 'Subscription file = %s' % args.instmts)
log_str = 'Exchange/Instrument/InstrumentCode:\n'
for instmt in subscription_instmts:
log_str += '%s/%s/%s\n' % (instmt.exchange_name, instmt.instmt_name, instmt.instmt_code)
Logger.info('[main]', log_str)
exch_gws = []
exch_gws.append(ExchGwBtccSpot(db_clients))
exch_gws.append(ExchGwBtccFuture(db_clients))
exch_gws.append(ExchGwBitmex(db_clients))
exch_gws.append(ExchGwBitfinex(db_clients))
exch_gws.append(ExchGwOkCoin(db_clients))
exch_gws.append(ExchGwKraken(db_clients))
exch_gws.append(ExchGwGdax(db_clients))
exch_gws.append(ExchGwBitstamp(db_clients))
exch_gws.append(ExchGwGatecoin(db_clients))
exch_gws.append(ExchGwQuoine(db_clients))
exch_gws.append(ExchGwPoloniex(db_clients))
exch_gws.append(ExchGwBittrex(db_clients))
threads = []
for exch in exch_gws:
for instmt in subscription_instmts:
if instmt.get_exchange_name() == exch.get_exchange_name():
Logger.info("[main]", "Starting instrument %s-%s..." % \
(instmt.get_exchange_name(), instmt.get_instmt_name()))
threads += exch.start(instmt)
