def main(config_path: str,
start_time: str = None,
end_time: str = None,
strategy_dir: str = '.'):
init_logging()
config = BacktestConfig.load(Path(config_path), Path(strategy_dir),
start_time, end_time)
engine = AlgoBacktester(config)
engine.run()
def ws_fh_main(create_fh, uri_scheme: str, instance_id: str, journal_path: str, db: str):
init_logging()
logger = logging.getLogger(__name__)
conn = connect_serenity_db()
conn.autocommit = True
cur = conn.cursor()
instr_cache = InstrumentCache(cur, TypeCodeCache(cur))
scheduler = NetworkScheduler()
registry = FeedHandlerRegistry()
fh = create_fh(scheduler, instr_cache, instance_id)
registry.register(fh)
for instrument in fh.get_instruments():
symbol = instrument.get_exchange_instrument_code()
# subscribe to FeedState in advance so we know when the Feed is ready to subscribe trades
class SubscribeTrades(Event):
def __init__(self, trade_symbol):
self.trade_symbol = trade_symbol
self.appender = None
def on_activate(self) -> bool:
if fh.get_state().get_value() == FeedHandlerState.LIVE:
feed = registry.get_feed(f'{uri_scheme}:{instance_id}:{self.trade_symbol}')
instrument_code = feed.get_instrument().get_exchange_instrument_code()
journal = Journal(Path(f'{journal_path}/{db}/{instrument_code}'))
self.appender = journal.create_appender()
trades = feed.get_trades()
Do(scheduler.get_network(), trades, lambda: self.on_trade_print(trades.get_value()))
return False
def on_trade_print(self, trade):
logger.info(trade)
self.appender.write_double(datetime.utcnow().timestamp())
self.appender.write_long(trade.get_trade_id())
self.appender.write_long(trade.get_trade_id())
self.appender.write_string(trade.get_instrument().get_exchange_instrument_code())
self.appender.write_short(1 if trade.get_side().get_type_code() == 'Buy' else 0)
self.appender.write_double(trade.get_qty())
self.appender.write_double(trade.get_price())
scheduler.get_network().connect(fh.get_state(), SubscribeTrades(symbol))
# async start the feedhandler
asyncio.ensure_future(fh.start())
# crash out on any exception
asyncio.get_event_loop().set_exception_handler(custom_asyncio_error_handler)
# go!
asyncio.get_event_loop().run_forever()
def main(config_path: str, strategy_dir: str, start_time: str, end_time: str):
init_logging()
config = toml.load(config_path)
strategy_module = config['strategy']['module']
strategy_class = config['strategy']['class']
loader = StrategyLoader(Path(strategy_dir))
strategy_instance = loader.load(strategy_module, strategy_class)
scheduler = HistoricNetworkScheduler.new_instance(start_time, end_time)
backtester = InvestmentStrategyBacktester(scheduler)
portfolio = backtester.run(strategy_instance, config)
scheduler.run()
for account in portfolio.get_accounts():
print(f'Account: [{account.get_name()}]')
print(f'\tcash: {account.get_cash_balance().get_balance()}')
for position in account.get_positions():
print(
f'\t{position.get_tradable().get_symbol()}: {position.get_qty()} shares ({position.get_notional()})'
)
def tickstore_admin(action: str,
db: str,
staging_dir: str = '/mnt/raid/data/behemoth/db',
connect_str: str = None,
db_prefix: str = None):
init_logging()
if action == 'reindex':
tickstore = LocalTickstore(Path(f'{staging_dir}/{db}'),
timestamp_column='time')
tickstore.index.reindex()
tickstore.close()
if action == 'strip_prefix':
tickstore = LocalTickstore(Path(f'{staging_dir}/{db}'),
timestamp_column='time')
tickstore.index.strip_prefix(db_prefix)
tickstore.close()
elif action == 'list':
tickstore = LocalTickstore(Path(f'{staging_dir}/{db}'),
timestamp_column='time')
for symbol in tickstore.index.symbols():
print(symbol)
for entry in tickstore.index.entries(symbol):
print(f'\t{entry}')
elif action == 'cloudsync':
local_tickstore = LocalTickstore(Path(f'{staging_dir}/{db}'),
timestamp_column='time')
cloud_tickstore = AzureBlobTickstore(connect_str, db)
for symbol in local_tickstore.index.symbols():
for entry in local_tickstore.index.entries(symbol):
logical_path = entry.path
ticks = local_tickstore.read(logical_path)
cloud_tickstore.insert(entry.symbol, entry.ts, ticks)
local_tickstore.close()
cloud_tickstore.close()
else:
raise Exception(f'Unknown action: {action}')
def upload_main(behemoth_path: str = '/behemoth', days_back: int = 1):
init_logging()
logger = logging.getLogger(__name__)
upload_date = datetime.datetime.utcnow().date() - datetime.timedelta(
days_back)
conn = connect_serenity_db()
conn.autocommit = True
cur = conn.cursor()
instr_cache = InstrumentCache(cur, TypeCodeCache(cur))
exchanges = {
'Phemex': 'PHEMEX_TRADES',
'CoinbasePro': 'COINBASE_PRO_TRADES'
}
for exchange, db in exchanges.items():
for instrument in instr_cache.get_all_exchange_instruments(exchange):
symbol = instrument.get_exchange_instrument_code()
path = Path(f'{behemoth_path}/journals/{db}/{symbol}')
journal = Journal(path)
try:
reader = journal.create_reader(upload_date)
length = reader.get_length()
records = []
while reader.get_pos() < length:
time = reader.read_double()
sequence = reader.read_long()
trade_id = reader.read_long()
product_id = reader.read_string()
side = 'buy' if reader.read_short() == 0 else 'sell'
size = reader.read_double()
price = reader.read_double()
record = {
'time': datetime.datetime.fromtimestamp(time),
'sequence': sequence,
'trade_id': trade_id,
'product_id': product_id,
'side': side,
'size': size,
'price': price
}
records.append(record)
if len(records) > 0:
logger.info(
f'uploading journaled {exchange}/{symbol} ticks to Behemoth for UTC date {str(upload_date)}'
)
df = pd.DataFrame(records)
df.set_index('time', inplace=True)
logger.info(f'extracted {len(df)} {symbol} trade records')
tickstore = LocalTickstore(
Path(Path(f'{behemoth_path}/db/{db}')), 'time')
tickstore.insert(symbol, BiTimestamp(upload_date), df)
tickstore.close()
logger.info(f'inserted {len(df)} {symbol} records')
else:
logger.info(
f'zero {exchange}/{symbol} ticks for UTC date {str(upload_date)}'
)
tickstore = LocalTickstore(
Path(Path(f'{behemoth_path}/db/{db}')), 'time')
tickstore.close()
except NoSuchJournalException:
logger.error(f'missing journal file: {path}')
def main(config_path: str, strategy_dir='.'):
init_logging()
engine = AlgoEngine(config_path, strategy_dir)
engine.start()
def ws_fh_main(create_fh,
uri_scheme: str,
instance_id: str,
journal_path: str,
db: str,
journal_books: bool = True,
include_symbol: str = '*'):
init_logging()
logger = logging.getLogger(__name__)
conn = connect_serenity_db()
conn.autocommit = True
cur = conn.cursor()
instr_cache = InstrumentCache(cur, TypeCodeCache(cur))
scheduler = RealtimeNetworkScheduler()
registry = FeedHandlerRegistry()
fh = create_fh(scheduler, instr_cache, include_symbol, instance_id)
registry.register(fh)
# register Prometheus metrics
trade_counter = Counter('serenity_trade_counter',
'Number of trade prints received by feedhandler')
book_update_counter = Counter(
'serenity_book_update_counter',
'Number of book updates received by feedhandler')
for instrument in fh.get_instruments():
symbol = instrument.get_exchange_instrument_code()
if not (symbol == include_symbol or include_symbol == '*'):
continue
# subscribe to FeedState in advance so we know when the Feed is ready to subscribe trades
class SubscribeTrades(Event):
def __init__(self, trade_symbol):
self.trade_symbol = trade_symbol
self.tx_writer = None
def on_activate(self) -> bool:
if fh.get_state().get_value() == FeedHandlerState.LIVE:
feed = registry.get_feed(
f'{uri_scheme}:{instance_id}:{self.trade_symbol}')
instrument_code = feed.get_instrument(
).get_exchange_instrument_code()
txlog = TransactionLog(
Path(f'{journal_path}/{db}_TRADES/{instrument_code}'))
self.tx_writer = txlog.create_writer()
trades = feed.get_trades()
Do(scheduler.get_network(), trades,
lambda: self.on_trade_print(trades.get_value()))
return False
def on_trade_print(self, trade):
trade_counter.inc()
logger.info(trade)
trade_msg = capnp_def.TradeMessage.new_message()
trade_msg.time = datetime.utcnow().timestamp()
trade_msg.tradeId = trade.get_trade_id()
trade_msg.side = capnp_def.Side.buy if trade.get_side(
) == Side.BUY else capnp_def.Side.sell
trade_msg.size = trade.get_qty()
trade_msg.price = trade.get_price()
self.tx_writer.append_msg(trade_msg)
if journal_books:
class SubscribeOrderBook(Event):
def __init__(self, trade_symbol):
self.trade_symbol = trade_symbol
self.tx_writer = None
def on_activate(self) -> bool:
if fh.get_state().get_value() == FeedHandlerState.LIVE:
feed = registry.get_feed(
f'{uri_scheme}:{instance_id}:{self.trade_symbol}')
instrument_code = feed.get_instrument(
).get_exchange_instrument_code()
txlog = TransactionLog(
Path(f'{journal_path}/{db}_BOOKS/{instrument_code}'
))
self.tx_writer = txlog.create_writer()
books = feed.get_order_books()
Do(scheduler.get_network(), books,
lambda: self.on_book_update(books.get_value()))
return False
def on_book_update(self, book: OrderBook):
book_update_counter.inc()
book_msg = capnp_def.Level1BookUpdateMessage.new_message()
book_msg.time = datetime.utcnow().timestamp()
if len(book.get_bids()) > 0:
book_msg.bestBidQty = book.get_best_bid().get_qty()
book_msg.bestBidPx = book.get_best_bid().get_px()
else:
book_msg.bestBidQty = 0
book_msg.bestBidPx = 0
if len(book.get_asks()) > 0:
book_msg.bestAskQty = book.get_best_ask().get_qty()
book_msg.bestAskPx = book.get_best_ask().get_px()
else:
book_msg.bestAskQty = 0
book_msg.bestAskPx = 0
self.tx_writer.append_msg(book_msg)
scheduler.get_network().connect(fh.get_state(),
SubscribeOrderBook(symbol))
scheduler.get_network().connect(fh.get_state(),
SubscribeTrades(symbol))
# launch the monitoring endpoint
start_http_server(8000)
# async start the feedhandler
asyncio.ensure_future(fh.start())
# crash out on any exception
asyncio.get_event_loop().set_exception_handler(
custom_asyncio_error_handler)
# go!
asyncio.get_event_loop().run_forever()
import datetime
import logging
import os
from pathlib import Path
import fire
import pandas as pd
from serenity.db.api import connect_serenity_db, InstrumentCache, TypeCodeCache
from serenity.marketdata.tickstore.api import LocalTickstore, BiTimestamp, AzureBlobTickstore
from serenity.marketdata.tickstore.journal import Journal, NoSuchJournalException
from serenity.utils import init_logging
init_logging()
logger = logging.getLogger(__name__)
# noinspection DuplicatedCode
def upload_main(behemoth_path: str = '/behemoth',
days_back: int = 1,
upload_start_date: str = None,
upload_end_date: str = None):
if upload_start_date is not None and upload_end_date is not None:
upload_start_date = datetime.datetime.strptime(upload_start_date,
'%Y-%m-%d').date()
upload_end_date = datetime.datetime.strptime(upload_end_date,
'%Y-%m-%d').date()
delta = upload_end_date - upload_start_date
for i in range(delta.days + 1):
