class MemoryLoader:
def __init__(self):
self.log_loader = LogLoader(self.order_book_tick, self.trade_tick,
self.funding_tick)
self.order_book = deque()
self.sell_trade = deque()
self.buy_trade = deque()
self.last_time = 0
def order_book_tick(self, time_stamp, order_book):
sell_min, sell_vol, sell_list, buy_max, buy_vol, buy_list = LogLoader.message_to_list(
order_book)
self.order_book.append(
OrderBook(time_stamp, sell_min, sell_vol, sell_list, buy_max,
buy_vol, buy_list))
self.new_tick(time_stamp)
def trade_tick(self, time_stamp, trade_buy, trade_sell):
# for trade in trade_buy:
# self.buy_trade.append(Trade(trade['price'], trade['volume']))
print(time_stamp, trade_buy, trade_sell)
def funding_tick(self, time_stamp, funding):
print(time_stamp, funding)
def new_tick(self, time):
if self.last_time + 1 != time:
#fillout
pass
def load_line(self, line):
self.log_loader.load_line(line)
def __init__(self):
self.log_loader = LogLoader(self.order_book_tick, self.trade_tick,
self.funding_tick)
self.order_book = deque()
self.sell_trade = deque()
self.buy_trade = deque()
self.last_time = 0
def test_loader_message_insert2(self):
print()
loader = LogLoader()
loader.on_message(data.partial_message)
loader.on_message(data.insert_message)
loader.on_message(data.insert_message)
loader.on_message(data.partial_message)
print(loader.get_market_depth())
def test_loader_message_delete1(self):
print('delete')
loader = LogLoader()
loader.on_message(data.partial_message)
loader.on_message(data.insert_message)
loader.on_message(data.delete_message)
print(loader.get_market_depth())
def order_book_tick(self, time_stamp, order_book):
sell_min, sell_vol, sell_list, buy_max, buy_vol, buy_list = LogLoader.message_to_list(
order_book)
self.order_book.append(
OrderBook(time_stamp, sell_min, sell_vol, sell_list, buy_max,
buy_vol, buy_list))
self.new_tick(time_stamp)
def insert_order_book_message(self, time, message):
sell_min, sell_vol, sell_list, buy_max, buy_vol, buy_list = LogLoader.message_to_list(
message)
self.insert_order_book(time, sell_min, sell_vol, sell_list, buy_max,
buy_vol, buy_list)
def __init__(self):
self.book_last_time = 0
self.funding_last_time = 0
self.trade_last_time = 0
self.log_db = None
self.log_loader = LogLoader(self.order_book_tick, self.trade_tick, self.funding_tick)
class DbLoader:
def __init__(self):
self.book_last_time = 0
self.funding_last_time = 0
self.trade_last_time = 0
self.log_db = None
self.log_loader = LogLoader(self.order_book_tick, self.trade_tick, self.funding_tick)
def open_db(self, db_file=None):
self.log_db = LogDb(db_file)
self.log_db.connect()
self.log_db.create_cursor()
self.log_db.create()
def close_db(self):
self.log_db.close()
def get_db(self):
return self.log_db
def load_line(self, line):
self.log_loader.load_line(line)
def load_lines(self, lines):
for line in lines:
self.log_loader.load_line(line)
def load_file(self, log_file):
print('Processs ' + log_file, end='')
try:
self.log_loader.load(log_file)
except EOFError as e:
print('error to process fileError EOF', e)
except Exception as e:
print('File process error SKIP', e)
def load_dir(self, log_dir ='/tmp'):
log_files = sorted(glob.glob(log_dir + '/' + '*.log'))
for file in log_files:
self.log_db.create_cursor()
self.load_file(file)
self.log_db.commit()
log_files = sorted(glob.glob(log_dir + '/' + '*.log.gz'))
for file in log_files:
self.log_db.create_cursor()
self.load_file(file)
self.log_db.commit()
def load_from_blobs(self, path=''):
log_storage = LogStorage()
log_storage.process_blob_dir(path, self.load_file)
def load_from_blob_by_date(self, year, month, day):
log_storage = LogStorage()
log_storage.process_blob_date_with_padding(year, month, day, self.load_file)
def order_book_tick(self, time_stamp, order_book):
if self.book_last_time != time_stamp:
self.log_db.insert_order_book_message(time_stamp, order_book)
self.book_last_time = time_stamp
def funding_tick(self, time_stamp, funding):
self.log_db.insert_funding(time_stamp, funding)
def trade_tick(self, time_stamp, buy_trade, sell_trade):
for price in buy_trade.keys():
self.log_db.insert_buy_trade(time_stamp, price, buy_trade[price])
for price in sell_trade.keys():
self.log_db.insert_sell_trade(time_stamp, price, sell_trade[price])
def test_loader_message_partial(self):
loader = LogLoader()
loader.on_message(data.partial_message)
def test_is_gizp(self):
loader = LogLoader()
result = loader._is_gzipfile("absc.log.gz")
self.assertEqual(result, True)
def test_loader_message(self):
loader = LogLoader()
loader.on_message(data.line)
def test_loader_message_indert(self):
loader = LogLoader()
loader.on_message(data.partial_message)
loader.on_message(data.insert_message)
def test_loader_message_delete(self):
loader = LogLoader()
loader.on_message(data.partial_message)
loader.on_message(data.delete_message)