def __init__(self, config_path: str, strategy_dir: str):
sys.path.append(strategy_dir)
with open(config_path, 'r') as config_yaml:
logger = logging.getLogger(__name__)
logger.info('Serenity starting up')
config = yaml.safe_load(config_yaml)
api_version = config['api-version']
if api_version != 'v1Beta':
raise ValueError(f'Unsupported API version: {api_version}')
self.engine_env = Environment(config['environment'])
instance_id = self.engine_env.getenv('FEED_INSTANCE', 'prod')
self.fh_registry = FeedHandlerRegistry()
logger.info('Connecting to Serenity database')
conn = connect_serenity_db()
conn.autocommit = True
cur = conn.cursor()
scheduler = NetworkScheduler()
instrument_cache = InstrumentCache(cur, TypeCodeCache(cur))
if 'feedhandlers' in config:
logger.info('Registering feedhandlers')
for feedhandler in config['feedhandlers']:
fh_name = feedhandler['exchange']
if fh_name == 'Phemex':
self.fh_registry.register(
PhemexFeedHandler(scheduler, instrument_cache,
instance_id))
elif fh_name == 'Binance':
self.fh_registry.register(
BinanceFeedHandler(scheduler, instrument_cache,
instance_id))
elif fh_name == 'CoinbasePro':
self.fh_registry.register(
CoinbaseProFeedHandler(scheduler, instrument_cache,
instance_id))
else:
raise ValueError(
f'Unsupported feedhandler type: {fh_name}')
self.strategies = []
for strategy in config['strategies']:
strategy_name = strategy['name']
self.logger.info(f'Loading strategy: {strategy_name}')
module = strategy['module']
strategy_class = strategy['strategy-class']
env = Environment(strategy['environment'])
module = importlib.import_module(module)
klass = getattr(module, strategy_class)
strategy_instance = klass()
md_service = FeedHandlerMarketdataService(
scheduler.get_network(), self.fh_registry, instance_id)
ctx = StrategyContext(scheduler, instrument_cache, md_service,
env.values)
self.strategies.append((strategy_instance, ctx))
def generate_tax_report():
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
analyzer = TradeAnalyzer(cur, type_code_cache, instrument_cache)
analyzer.run_analysis(['BTC-USD', 'ETH-USD'], 2019, 0.35)
def ws_fh_main(create_fh, uri_scheme: str, instance_id: str, journal_path: str, db: str):
init_logging()
logger = logging.getLogger(__name__)
conn = connect_serenity_db()
conn.autocommit = True
cur = conn.cursor()
instr_cache = InstrumentCache(cur, TypeCodeCache(cur))
scheduler = NetworkScheduler()
registry = FeedHandlerRegistry()
fh = create_fh(scheduler, instr_cache, instance_id)
registry.register(fh)
for instrument in fh.get_instruments():
symbol = instrument.get_exchange_instrument_code()
# subscribe to FeedState in advance so we know when the Feed is ready to subscribe trades
class SubscribeTrades(Event):
def __init__(self, trade_symbol):
self.trade_symbol = trade_symbol
self.appender = None
def on_activate(self) -> bool:
if fh.get_state().get_value() == FeedHandlerState.LIVE:
feed = registry.get_feed(f'{uri_scheme}:{instance_id}:{self.trade_symbol}')
instrument_code = feed.get_instrument().get_exchange_instrument_code()
journal = Journal(Path(f'{journal_path}/{db}/{instrument_code}'))
self.appender = journal.create_appender()
trades = feed.get_trades()
Do(scheduler.get_network(), trades, lambda: self.on_trade_print(trades.get_value()))
return False
def on_trade_print(self, trade):
logger.info(trade)
self.appender.write_double(datetime.utcnow().timestamp())
self.appender.write_long(trade.get_trade_id())
self.appender.write_long(trade.get_trade_id())
self.appender.write_string(trade.get_instrument().get_exchange_instrument_code())
self.appender.write_short(1 if trade.get_side().get_type_code() == 'Buy' else 0)
self.appender.write_double(trade.get_qty())
self.appender.write_double(trade.get_price())
scheduler.get_network().connect(fh.get_state(), SubscribeTrades(symbol))
# async start the feedhandler
asyncio.ensure_future(fh.start())
# crash out on any exception
asyncio.get_event_loop().set_exception_handler(custom_asyncio_error_handler)
# go!
asyncio.get_event_loop().run_forever()
def backfill_coinbase():
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
exch_service = ExchangeEntityService(cur, type_code_cache,
instrument_cache)
# create a default account for Coinbase
exchange = type_code_cache.get_by_code(Exchange, 'Coinbase')
account = exch_service.get_or_create_account(
ExchangeAccount(0, exchange, 'default'))
# create a BTC-USD symbol for Coinbase
product_id = 'BTC-USD'
instrument_type = type_code_cache.get_by_code(InstrumentType,
'CurrencyPair')
instrument = instrument_cache.get_or_create_instrument(
product_id, instrument_type)
exchange_instrument = instrument_cache.get_or_create_exchange_instrument(
product_id, instrument, 'Coinbase')
# synthesize market orders and fills for Coinbase purchases
side = type_code_cache.get_by_code(Side, 'Buy')
order_type = type_code_cache.get_by_code(OrderType, 'Market')
tif = type_code_cache.get_by_code(TimeInForce, 'Day')
# noinspection DuplicatedCode
def create_exchange_order_and_fill(price, quantity, fees, order_id,
create_time):
order = ExchangeOrder(0, exchange, exchange_instrument, order_type,
account, side, tif, order_id, price, quantity,
create_time)
order = exch_service.get_or_create_exchange_order(order)
conn.commit()
fill = ExchangeFill(0, price, quantity, fees, order_id, create_time)
fill.set_order(order)
exch_service.get_or_create_exchange_fill(fill)
conn.commit()
# investment purchase
create_exchange_order_and_fill(265.39, 1.13, 2.9993, 1,
datetime(2015, 1, 26))
# residual left in account after payment by Bitcoin
create_exchange_order_and_fill(238.47, 2.1 - 1.68136, 5.013, 2,
datetime(2015, 2, 13))
# investment purchase
create_exchange_order_and_fill(249.05, 0.5, 1.25, 3, datetime(2015, 2, 14))
def upload_main(behemoth_path: str = '/behemoth', days_back: int = 1):
init_logging()
logger = logging.getLogger(__name__)
upload_date = datetime.datetime.utcnow().date() - datetime.timedelta(
days_back)
conn = connect_serenity_db()
conn.autocommit = True
cur = conn.cursor()
instr_cache = InstrumentCache(cur, TypeCodeCache(cur))
exchanges = {
'Phemex': 'PHEMEX_TRADES',
'CoinbasePro': 'COINBASE_PRO_TRADES'
}
for exchange, db in exchanges.items():
for instrument in instr_cache.get_all_exchange_instruments(exchange):
symbol = instrument.get_exchange_instrument_code()
path = Path(f'{behemoth_path}/journals/{db}/{symbol}')
journal = Journal(path)
try:
reader = journal.create_reader(upload_date)
length = reader.get_length()
records = []
while reader.get_pos() < length:
time = reader.read_double()
sequence = reader.read_long()
trade_id = reader.read_long()
product_id = reader.read_string()
side = 'buy' if reader.read_short() == 0 else 'sell'
size = reader.read_double()
price = reader.read_double()
record = {
'time': datetime.datetime.fromtimestamp(time),
'sequence': sequence,
'trade_id': trade_id,
'product_id': product_id,
'side': side,
'size': size,
'price': price
}
records.append(record)
if len(records) > 0:
logger.info(
f'uploading journaled {exchange}/{symbol} ticks to Behemoth for UTC date {str(upload_date)}'
)
df = pd.DataFrame(records)
df.set_index('time', inplace=True)
logger.info(f'extracted {len(df)} {symbol} trade records')
tickstore = LocalTickstore(
Path(Path(f'{behemoth_path}/db/{db}')), 'time')
tickstore.insert(symbol, BiTimestamp(upload_date), df)
tickstore.close()
logger.info(f'inserted {len(df)} {symbol} records')
else:
logger.info(
f'zero {exchange}/{symbol} ticks for UTC date {str(upload_date)}'
)
tickstore = LocalTickstore(
Path(Path(f'{behemoth_path}/db/{db}')), 'time')
tickstore.close()
except NoSuchJournalException:
logger.error(f'missing journal file: {path}')
def backfill_coinbasepro(api_key: str, api_secret: str, api_passphrase: str):
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
exch_service = ExchangeEntityService(cur, type_code_cache,
instrument_cache)
auth_client = coinbasepro.AuthenticatedClient(key=api_key,
secret=api_secret,
passphrase=api_passphrase)
# Coinbase Pro has a notion of account per currency for tracking balances, so we want to pull
# out what it calls the profile, which is the parent exchange account
profile_set = set()
for account in auth_client.get_accounts():
profile_set.add(account['profile_id'])
exchange = type_code_cache.get_by_code(Exchange, "CoinbasePro")
account_by_profile_id = {}
for profile in profile_set:
account = exch_service.get_or_create_account(
ExchangeAccount(0, exchange, profile))
account_by_profile_id[profile] = account
# load up all the orders
for order in auth_client.get_orders(status=['done']):
order_uuid = order['id']
# market orders have no price
if 'price' in order:
price = order['price']
else:
price = None
# market orders that specify "funds" have no size
if 'size' in order:
size = order['size']
else:
size = order['filled_size']
exchange_account = account_by_profile_id[order['profile_id']]
instrument_type = type_code_cache.get_by_code(InstrumentType,
'CurrencyPair')
instrument = instrument_cache.get_or_create_instrument(
order['product_id'], instrument_type)
exchange_instrument = instrument_cache.get_or_create_exchange_instrument(
order['product_id'], instrument, exchange.get_type_code())
side = type_code_cache.get_by_code(Side, order['side'].capitalize())
if order['type'] is None:
order['type'] = 'Market'
order_type = type_code_cache.get_by_code(OrderType,
order['type'].capitalize())
if 'time_in_force' in order:
tif = type_code_cache.get_by_code(TimeInForce,
order['time_in_force'])
else:
tif = type_code_cache.get_by_code(TimeInForce, 'Day')
create_time = order['created_at']
order = ExchangeOrder(0, exchange, exchange_instrument, order_type,
exchange_account, side, tif, order_uuid, price,
size, create_time)
exch_service.get_or_create_exchange_order(order)
conn.commit()
# load up all the fills, linking back to the orders
for product in ['BTC-USD', 'ETH-BTC']:
for fill in auth_client.get_fills(product_id=product):
order_id = fill['order_id']
trade_id = fill['trade_id']
price = fill['price']
size = fill['size']
fees = fill['fee']
create_time = fill['created_at']
order = exch_service.get_entity_by_ak(
ExchangeOrder, (exchange.get_type_code(), order_id))
fill = ExchangeFill(0, price, size, fees, trade_id, create_time)
fill.set_order(order)
exch_service.get_or_create_exchange_fill(fill)
conn.commit()
import yfinance
from serenity.db import connect_serenity_db
conn = connect_serenity_db()
cur = conn.cursor()
def backfill_symbol(symbol):
ticker = yfinance.Ticker(symbol)
marks = ticker.history(period="max")
cur.execute(
"SELECT instrument_id FROM serenity.instrument WHERE instrument_code = %s",
(symbol, ))
instrument_id = cur.fetchone()[0]
cur.execute(
"SELECT mark_type_id FROM serenity.mark_type WHERE mark_type_code = %s",
("YahooDailyClose", ))
mark_type_id = cur.fetchone()[0]
for index, row in marks.iterrows():
cur.execute(
"INSERT INTO serenity.instrument_mark (instrument_id, mark_type_id, mark_time, mark) "
"VALUES (%s, %s, %s, %s)",
(instrument_id, mark_type_id, index, float(row['Close'])))
conn.commit()
def backfill_gemini(gemini_api_key: str, gemini_api_secret: str):
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
exch_service = ExchangeEntityService(cur, type_code_cache,
instrument_cache)
client = gemini.PrivateClient(gemini_api_key, gemini_api_secret)
for exchange_symbol in ('BTCUSD', 'ETHBTC', 'ZECBTC'):
instrument_symbol = exchange_symbol[0:3] + '-' + exchange_symbol[3:]
instrument_type = type_code_cache.get_by_code(InstrumentType,
'CurrencyPair')
instrument = instrument_cache.get_or_create_instrument(
instrument_symbol, instrument_type)
exchange_instrument = instrument_cache.get_or_create_exchange_instrument(
exchange_symbol.lower(), instrument, 'Gemini')
conn.commit()
exchange = type_code_cache.get_by_code(Exchange, 'Gemini')
for trade in client.get_past_trades(exchange_symbol):
fill_price = trade['price']
quantity = trade['amount']
fees = trade['fee_amount']
side = type_code_cache.get_by_code(Side, trade['type'])
trade_id = trade['tid']
order_uuid = trade['order_id']
create_time_ms = trade['timestampms']
create_time = datetime.utcfromtimestamp(create_time_ms // 1000).\
replace(microsecond=create_time_ms % 1000 * 1000)
# because we cannot get historical exchange orders past 7 days, we need to synthesize limit orders
exchange_account = ExchangeAccount(0, exchange, 'default')
exchange_account = exch_service.get_or_create_account(
exchange_account)
order_type = type_code_cache.get_by_code(OrderType, 'Limit')
tif = type_code_cache.get_by_code(TimeInForce, 'GTC')
order = ExchangeOrder(0, exchange, exchange_instrument, order_type,
exchange_account, side, tif, order_uuid,
fill_price, quantity, create_time)
order = exch_service.get_or_create_exchange_order(order)
conn.commit()
# create the fills and insert, linking back to the synthetic order
fill = ExchangeFill(0, fill_price, quantity, fees, trade_id,
create_time)
fill.set_order(order)
exch_service.get_or_create_exchange_fill(fill)
conn.commit()
for transfer in client.api_query('/v1/transfers', {}):
transfer_type = type_code_cache.get_by_code(ExchangeTransferType,
transfer['type'])
transfer_method = type_code_cache.get_by_code(ExchangeTransferMethod,
"Blockchain")
currency = instrument_cache.get_or_create_currency(
transfer['currency'])
quantity = transfer['amount']
transfer_ref = transfer['txHash']
transfer_time_ms = transfer['timestampms']
transfer_time = datetime.utcfromtimestamp(transfer_time_ms // 1000). \
replace(microsecond=transfer_time_ms % 1000 * 1000)
transfer = ExchangeTransfer(0, exchange, transfer_type,
transfer_method, currency, quantity,
transfer_ref, transfer_time)
exch_service.get_or_create_exchange_transfer(transfer)
conn.commit()
