def test_list_price_info(self):
db = LogDb(db_name='/bitlog/bitlog.db')
db.connect()
time = None # 0
market_order_sell = None # 1
market_order_buy = None # 2
fix_order_sell = None # 3
fix_order_sell_time = None # 4
fix_order_buy = None # 5
fix_order_buy_time = None # 6
org_price = OrderPrices()
line_count = 0
block_count = 0
for line in db.list_price():
price = OrderPrices()
price.set_price_record(line)
if ((price.market_order_sell != org_price.market_order_sell)
or (price.market_order_buy != org_price.market_order_buy)
or (price.fix_order_sell != org_price.fix_order_sell)
or (price.fix_order_buy != org_price.fix_order_buy)):
print(price, ',')
block_count += 1
line_count += 1
org_price.set_price_record(line)
print('line', line_count, 'block_count', block_count)
def test_create(self):
log_db = LogDb()
log_db.connect()
log_db.create_cursor()
log_db.create()
log_db.commit()
def test_update_all_q_values(self):
db = LogDb('/bitlog/bitlog.db')
db.connect()
db.create_cursor()
db.update_all_q()
db.commit()
def test_calc_best_actions(self):
db = LogDb()
print(db._calc_best_actions(0))
print(db._calc_best_actions(1))
print(db._calc_best_actions(2))
print(db._calc_best_actions(3))
print(db._calc_best_actions(4))
def test_message_to_list_zip(self):
log_db = LogDb()
message = [1000, 2, 3, 400]
m2 = log_db.list_to_zip_string(message)
m3 = log_db.zip_string_to_list(m2)
print("row->", message, "zip->", m2, "unzip->", m3)
assert (message == m3)
def test_import_db(self):
db = LogDb()
db.connect()
db.create_cursor()
db.import_db()
print(db.get_db_info())
def test_select_low_price(self):
db = LogDb('/bitlog/bitlog.db')
db.connect()
db.create_cursor()
start, _ = db.get_db_info()
r = db.select_lowest_price_time(start)
print(start, r)
def test_load_tf_records_with_time(self):
db = LogDb()
db.connect()
db.create_cursor()
db.import_db()
time = 1552910400
PriceBoardDB.export_db_to_blob_with_time(db, time, width=600, root_dir='/tmp/')
def test_create_low_q_sequcen(self):
db = LogDb('/bitlog/bitlog.db')
db.connect()
db.create_cursor()
start, _ = db.get_db_info()
start_price = 7220
action = ACTION.SELL
db.create_q_sequence(start_time=start,
action=action,
start_price=start_price)
def test_print_order_book_line(self):
db = LogDb('/tmp/bitlog.db')
db.connect()
db.create_cursor()
line = db.select_order_book(1553099021)
print(line)
line = db.select_order_book(1553099022)
print(line)
line = db.select_order_book(1553099023)
print(line)
def db2blob(year, month, day, root_dir):
db_file = '/tmp/{:04d}-{:02d}-{:02d}.db'.format(year, month, day)
db = LogDb()
db.connect()
db.create_cursor()
db.import_db(db_file)
date = '{:04d}-{:02d}-{:02d}'.format(year, month, day)
start_time = int(time_sec(date + 'T00:00:00.00'))
end_time = start_time + 24 * 60 * 60
PriceBoardDB.export_board_to_blob(db_object=db, start_time=start_time, end_time=end_time, root_dir=root_dir)
db.close()
def test_decode_list(self):
db = LogDb()
l = [99991, 99992, 99993, 99994, 99995, 99991, 99992, 99993, 99994, 99995, 99991, 99992, 99993, 99994, 99995]
print(l)
print(len(l))
b = db.list_to_bin(l)
print(b)
print('compressize=>', len(b))
l2 = db.bin_to_list(b)
print(l2)
print(len(l2))
zs = db.list_to_zip_string(l)
print(len(zs))
print(zs)
def test_load_tf_records_with_time_from_file(self):
db = LogDb('/tmp/bitlog.db')
db.connect()
db.create_cursor()
time = 1552957717
PriceBoardDB.export_db_to_blob_with_time(db, time, width=600, root_dir='/tmp/')
def test_load_from_db_600(self):
end_time = self.calc_end_time()
t = end_time
db = LogDb('/tmp/bitlog.db')
db.connect()
db.create_cursor()
writer = PriceBoardDB.get_tf_writer('/tmp/600rec.tfrecords')
retry = 600
while retry:
board = PriceBoardDB.load_from_connected_db(t-retry, db)
if board:
board.save_tf_to_writer(writer)
break
retry -= 1
db.close()
writer.close()
def test_crate_db_insert_q(self):
db = LogDb() # create on memory
db.connect()
db.create_cursor()
db.create()
db.commit()
q = QValue()
q.q[ACTION.NOP] = 1
q.q[ACTION.BUY] = 1
q.q[ACTION.SELL] = 1
q.q[ACTION.SELL_NOW] = 1
q.q[ACTION.BUY_NOW] = 1
db.insert_q(100, 101, ACTION.BUY, q)
def test_crate_db(self):
db = LogDb() # create on memory
db.connect()
db.create_cursor()
db.create()
db.commit()
def test_select_q(self):
db = LogDb('/bitlog/bitlog.db')
db.connect()
db.create_cursor()
q = db.select_q(0, 0, 0)
print(q)
q = db.select_q(0, 0, 1)
print(q)
q = db.select_q(1575501187, 1575501187, 2)
print(q)
q = db.select_q(1575542888, 1575542888, 1)
print(q)
q = db.select_q(1575542888, 1575542888, 2)
print(q)
q = db.select_q(1575542888, 1575542888, 3)
print(q)
q = db.select_q(1575542888, 1575542888, 4)
print(q)
def test_db_info(self):
db = LogDb("/tmp/bitlog.db")
db.connect()
db.get_db_info()
def test_best_action(self):
db = LogDb()
action = db.best_action(market_order_buy=0,
market_order_buy_f=0,
market_order_sell=0,
market_order_sell_f=0,
fix_order_buy=0,
fix_order_buy_f=0,
fix_order_sell=0,
fix_order_sell_f=0)
self.assertTrue(action == constant.ACTION.NOP)
action = db.best_action(market_order_buy=None,
market_order_buy_f=None,
market_order_sell=None,
market_order_sell_f=None,
fix_order_buy=None,
fix_order_buy_f=None,
fix_order_sell=None,
fix_order_sell_f=None)
self.assertTrue(action == constant.ACTION.NOP)
action = db.best_action(market_order_buy=10,
market_order_buy_f=None,
market_order_sell=None,
market_order_sell_f=14.5,
fix_order_buy=None,
fix_order_buy_f=None,
fix_order_sell=None,
fix_order_sell_f=None)
print(action)
self.assertTrue(action == constant.ACTION.BUY_NOW)
action = db.best_action(market_order_buy=10,
market_order_buy_f=0,
market_order_sell=0,
market_order_sell_f=14.5,
fix_order_buy=10,
fix_order_buy_f=0,
fix_order_sell=0,
fix_order_sell_f=14)
self.assertTrue(action == constant.ACTION.BUY)
action = db.best_action(market_order_buy=0,
market_order_buy_f=0,
market_order_sell=0,
market_order_sell_f=0,
fix_order_buy=0,
fix_order_buy_f=9,
fix_order_sell=9.5,
fix_order_sell_f=0) # margin is small
print(action)
self.assertTrue(action == constant.ACTION.NOP)
action = db.best_action(market_order_buy=0,
market_order_buy_f=0,
market_order_sell=0,
market_order_sell_f=0,
fix_order_buy=0,
fix_order_buy_f=9,
fix_order_sell=12,
fix_order_sell_f=0)
print(action)
self.assertTrue(action == constant.ACTION.SELL)
action = db.best_action(market_order_buy=10,
market_order_buy_f=0,
market_order_sell=0,
market_order_sell_f=14.5,
fix_order_buy=0,
fix_order_buy_f=0,
fix_order_sell=0,
fix_order_sell_f=0)
self.assertTrue(action == constant.ACTION.BUY_NOW)
action = db.best_action(market_order_buy=0,
market_order_buy_f=10,
market_order_sell=14.5,
market_order_sell_f=0,
fix_order_buy=0,
fix_order_buy_f=0,
fix_order_sell=0,
fix_order_sell_f=0)
self.assertTrue(action == constant.ACTION.SELL_NOW)
action = db.best_action(market_order_buy=0,
market_order_buy_f=3890,
market_order_sell=3899,
market_order_sell_f=0,
fix_order_buy=0,
fix_order_buy_f=0,
fix_order_sell=None,
fix_order_sell_f=0)
self.assertTrue(action == constant.ACTION.SELL_NOW)
action = db.best_action(market_order_buy=0,
market_order_buy_f=0,
market_order_sell=0,
market_order_sell_f=0,
fix_order_buy=100,
fix_order_buy_f=99,
fix_order_sell=100,
fix_order_sell_f=101)
print(action)
self.assertTrue(action == constant.ACTION.SELL)
def test_select_hi_price(self):
db = LogDb('/bitlog/bitlog.db')
db.connect()
db.create_cursor()
start, _ = db.get_db_info()
p = db.select_center_price(start)
r = db.select_highest_price_time(start)
print(start, p, r)
start += 500
p = db.select_center_price(start)
r = db.select_highest_price_time(start)
print(start, p, r)
start += 500
p = db.select_center_price(start)
r = db.select_highest_price_time(start)
print(start, p, r)
start += 500
p = db.select_center_price(start)
r = db.select_highest_price_time(start)
print(start, p, r)
start += 5000
p = db.select_center_price(start)
r = db.select_highest_price_time(start)
print(start, p, r)
def update_db(year, month, day):
db_file = '/tmp/{:04d}-{:02d}-{:02d}.db'.format(year, month, day)
# updae db
db = LogDb(db_file)
db.connect()
db.create_cursor()
db.update_all_order_prices()
db.update_all_best_action()
#db.skip_nop_close_to_action()
db.close()
def test_update_q_on_nop(self):
db = LogDb('/bitlog/bitlog.db')
db.connect()
db.create_cursor()
db.update_q_on_nop()
def test_select_q(self):
db = LogDb('/bitlog/bitlog.db')
db.connect()
db.create_cursor()
db.create()
start, end = db.get_db_info()
#q = db.select_q_values(start, ACTION.NOP)
#q = db.select_q_values(start, ACTION.BUY)
#q = db.select_q_values(start, ACTION.BUY_NOW)
q = db.select_q_values(start, ACTION.SELL)
print(q)
q = db.select_q_values(start, ACTION.SELL_NOW)
print(q)
class DbLoader:
def __init__(self):
self.book_last_time = 0
self.funding_last_time = 0
self.trade_last_time = 0
self.log_db = None
self.log_loader = LogLoader(self.order_book_tick, self.trade_tick, self.funding_tick)
def open_db(self, db_file=None):
self.log_db = LogDb(db_file)
self.log_db.connect()
self.log_db.create_cursor()
self.log_db.create()
def close_db(self):
self.log_db.close()
def get_db(self):
return self.log_db
def load_line(self, line):
self.log_loader.load_line(line)
def load_lines(self, lines):
for line in lines:
self.log_loader.load_line(line)
def load_file(self, log_file):
print('Processs ' + log_file, end='')
try:
self.log_loader.load(log_file)
except EOFError as e:
print('error to process fileError EOF', e)
except Exception as e:
print('File process error SKIP', e)
def load_dir(self, log_dir ='/tmp'):
log_files = sorted(glob.glob(log_dir + '/' + '*.log'))
for file in log_files:
self.log_db.create_cursor()
self.load_file(file)
self.log_db.commit()
log_files = sorted(glob.glob(log_dir + '/' + '*.log.gz'))
for file in log_files:
self.log_db.create_cursor()
self.load_file(file)
self.log_db.commit()
def load_from_blobs(self, path=''):
log_storage = LogStorage()
log_storage.process_blob_dir(path, self.load_file)
def load_from_blob_by_date(self, year, month, day):
log_storage = LogStorage()
log_storage.process_blob_date_with_padding(year, month, day, self.load_file)
def order_book_tick(self, time_stamp, order_book):
if self.book_last_time != time_stamp:
self.log_db.insert_order_book_message(time_stamp, order_book)
self.book_last_time = time_stamp
def funding_tick(self, time_stamp, funding):
self.log_db.insert_funding(time_stamp, funding)
def trade_tick(self, time_stamp, buy_trade, sell_trade):
for price in buy_trade.keys():
self.log_db.insert_buy_trade(time_stamp, price, buy_trade[price])
for price in sell_trade.keys():
self.log_db.insert_sell_trade(time_stamp, price, sell_trade[price])
def test_list_db_q(self):
db = LogDb() # create on memory
db.connect()
db.create_cursor()
db.create()
db.commit()
q = QValue()
q.q[ACTION.NOP] = 1
q.q[ACTION.BUY] = 2
q.q[ACTION.SELL] = 3
q.q[ACTION.SELL_NOW] = 4
q.q[ACTION.BUY_NOW] = 5
db.insert_q(100, 0, ACTION.BUY, q)
db.insert_q(101, 100, ACTION.BUY, q)
db.insert_q(102, 100, ACTION.BUY, q)
db.insert_q(103, 100, ACTION.BUY, q)
db.insert_q(101, 101, ACTION.BUY, q)
db.commit()
def calc_end_time(self):
db = LogDb("/tmp/bitlog.db")
db.connect()
db.create_cursor()
return int(db.get_db_info()[1])
def open_db(self, db_file=None):
self.log_db = LogDb(db_file)
self.log_db.connect()
self.log_db.create_cursor()
self.log_db.create()
def test_create_q_values(self):
db = LogDb('/bitlog/bitlog.db')
db.connect()
db.create_cursor()
db.create()
start, end = db.get_db_info()
print('start/end', start, end)
price = db.list_price(start_time=start, end_time=start + 150)
print(len(price))
q_seq = db.create_q_sequence(start_time=start,
action=ACTION.SELL,
start_price=7245)
q_seq.dump_q()
from log.logdb import LogDb
import os
import sys
if __name__ == "__main__":
db_file = os.sep + 'tmp' + os.sep + 'bitlog.db'
if len(sys.argv) == 2:
db_file = sys.argv[1]
print(db_file)
db = LogDb(db_file)
db.connect()
db.create_cursor()
db.update_all_order_prices(True)
#db.update_all_best_action(True)
db.commit()
skip_number = db.skip_nop_close_to_action()
print('skip rec->', skip_number)
db.commit()
db.update_all_q()
db.commit()
db.close()
def test_create_q_and_insert(self):
db = LogDb('/bitlog/bitlog.db')
db.connect()
db.create_cursor()
db.create()
start, end = db.get_db_info()
print('start/end', start, end)
db.insert_updated_q()
