def backfill_coinbase():
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
exch_service = ExchangeEntityService(cur, type_code_cache,
instrument_cache)
# create a default account for Coinbase
exchange = type_code_cache.get_by_code(Exchange, 'Coinbase')
account = exch_service.get_or_create_account(
ExchangeAccount(0, exchange, 'default'))
# create a BTC-USD symbol for Coinbase
product_id = 'BTC-USD'
instrument_type = type_code_cache.get_by_code(InstrumentType,
'CurrencyPair')
instrument = instrument_cache.get_or_create_instrument(
product_id, instrument_type)
exchange_instrument = instrument_cache.get_or_create_exchange_instrument(
product_id, instrument, 'Coinbase')
# synthesize market orders and fills for Coinbase purchases
side = type_code_cache.get_by_code(Side, 'Buy')
order_type = type_code_cache.get_by_code(OrderType, 'Market')
tif = type_code_cache.get_by_code(TimeInForce, 'Day')
# noinspection DuplicatedCode
def create_exchange_order_and_fill(price, quantity, fees, order_id,
create_time):
order = ExchangeOrder(0, exchange, exchange_instrument, order_type,
account, side, tif, order_id, price, quantity,
create_time)
order = exch_service.get_or_create_exchange_order(order)
conn.commit()
fill = ExchangeFill(0, price, quantity, fees, order_id, create_time)
fill.set_order(order)
exch_service.get_or_create_exchange_fill(fill)
conn.commit()
# investment purchase
create_exchange_order_and_fill(265.39, 1.13, 2.9993, 1,
datetime(2015, 1, 26))
# residual left in account after payment by Bitcoin
create_exchange_order_and_fill(238.47, 2.1 - 1.68136, 5.013, 2,
datetime(2015, 2, 13))
# investment purchase
create_exchange_order_and_fill(249.05, 0.5, 1.25, 3, datetime(2015, 2, 14))
def __init__(self, config_path: str, strategy_dir: str):
sys.path.append(strategy_dir)
with open(config_path, 'r') as config_yaml:
logger = logging.getLogger(__name__)
logger.info('Serenity starting up')
config = yaml.safe_load(config_yaml)
api_version = config['api-version']
if api_version != 'v1Beta':
raise ValueError(f'Unsupported API version: {api_version}')
self.engine_env = Environment(config['environment'])
instance_id = self.engine_env.getenv('FEED_INSTANCE', 'prod')
self.fh_registry = FeedHandlerRegistry()
logger.info('Connecting to Serenity database')
conn = connect_serenity_db()
conn.autocommit = True
cur = conn.cursor()
scheduler = NetworkScheduler()
instrument_cache = InstrumentCache(cur, TypeCodeCache(cur))
if 'feedhandlers' in config:
logger.info('Registering feedhandlers')
for feedhandler in config['feedhandlers']:
fh_name = feedhandler['exchange']
if fh_name == 'Phemex':
self.fh_registry.register(
PhemexFeedHandler(scheduler, instrument_cache,
instance_id))
elif fh_name == 'Binance':
self.fh_registry.register(
BinanceFeedHandler(scheduler, instrument_cache,
instance_id))
elif fh_name == 'CoinbasePro':
self.fh_registry.register(
CoinbaseProFeedHandler(scheduler, instrument_cache,
instance_id))
else:
raise ValueError(
f'Unsupported feedhandler type: {fh_name}')
self.strategies = []
for strategy in config['strategies']:
strategy_name = strategy['name']
self.logger.info(f'Loading strategy: {strategy_name}')
module = strategy['module']
strategy_class = strategy['strategy-class']
env = Environment(strategy['environment'])
module = importlib.import_module(module)
klass = getattr(module, strategy_class)
strategy_instance = klass()
md_service = FeedHandlerMarketdataService(
scheduler.get_network(), self.fh_registry, instance_id)
ctx = StrategyContext(scheduler, instrument_cache, md_service,
env.values)
self.strategies.append((strategy_instance, ctx))
def generate_tax_report():
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
analyzer = TradeAnalyzer(cur, type_code_cache, instrument_cache)
analyzer.run_analysis(['BTC-USD', 'ETH-USD'], 2019, 0.35)
def ws_fh_main(create_fh, uri_scheme: str, instance_id: str, journal_path: str, db: str):
init_logging()
logger = logging.getLogger(__name__)
conn = connect_serenity_db()
conn.autocommit = True
cur = conn.cursor()
instr_cache = InstrumentCache(cur, TypeCodeCache(cur))
scheduler = NetworkScheduler()
registry = FeedHandlerRegistry()
fh = create_fh(scheduler, instr_cache, instance_id)
registry.register(fh)
for instrument in fh.get_instruments():
symbol = instrument.get_exchange_instrument_code()
# subscribe to FeedState in advance so we know when the Feed is ready to subscribe trades
class SubscribeTrades(Event):
def __init__(self, trade_symbol):
self.trade_symbol = trade_symbol
self.appender = None
def on_activate(self) -> bool:
if fh.get_state().get_value() == FeedHandlerState.LIVE:
feed = registry.get_feed(f'{uri_scheme}:{instance_id}:{self.trade_symbol}')
instrument_code = feed.get_instrument().get_exchange_instrument_code()
journal = Journal(Path(f'{journal_path}/{db}/{instrument_code}'))
self.appender = journal.create_appender()
trades = feed.get_trades()
Do(scheduler.get_network(), trades, lambda: self.on_trade_print(trades.get_value()))
return False
def on_trade_print(self, trade):
logger.info(trade)
self.appender.write_double(datetime.utcnow().timestamp())
self.appender.write_long(trade.get_trade_id())
self.appender.write_long(trade.get_trade_id())
self.appender.write_string(trade.get_instrument().get_exchange_instrument_code())
self.appender.write_short(1 if trade.get_side().get_type_code() == 'Buy' else 0)
self.appender.write_double(trade.get_qty())
self.appender.write_double(trade.get_price())
scheduler.get_network().connect(fh.get_state(), SubscribeTrades(symbol))
# async start the feedhandler
asyncio.ensure_future(fh.start())
# crash out on any exception
asyncio.get_event_loop().set_exception_handler(custom_asyncio_error_handler)
# go!
asyncio.get_event_loop().run_forever()
def __init__(self, scheduler: NetworkScheduler, instrument_cache: InstrumentCache,
instance_id: str):
self.scheduler = scheduler
self.instrument_cache = instrument_cache
self.type_code_cache = instrument_cache.get_type_code_cache()
self.instance_id = instance_id
self.buy_code = self.type_code_cache.get_by_code(Side, 'Buy')
self.sell_code = self.type_code_cache.get_by_code(Side, 'Sell')
self.instruments = []
self.known_instrument_ids = {}
self.price_scaling = {}
self._load_instruments()
self.state = MutableSignal(FeedHandlerState.INITIALIZING)
self.scheduler.get_network().attach(self.state)
def upload_main(behemoth_path: str = '/behemoth', days_back: int = 1):
init_logging()
logger = logging.getLogger(__name__)
upload_date = datetime.datetime.utcnow().date() - datetime.timedelta(
days_back)
conn = connect_serenity_db()
conn.autocommit = True
cur = conn.cursor()
instr_cache = InstrumentCache(cur, TypeCodeCache(cur))
exchanges = {
'Phemex': 'PHEMEX_TRADES',
'CoinbasePro': 'COINBASE_PRO_TRADES'
}
for exchange, db in exchanges.items():
for instrument in instr_cache.get_all_exchange_instruments(exchange):
symbol = instrument.get_exchange_instrument_code()
path = Path(f'{behemoth_path}/journals/{db}/{symbol}')
journal = Journal(path)
try:
reader = journal.create_reader(upload_date)
length = reader.get_length()
records = []
while reader.get_pos() < length:
time = reader.read_double()
sequence = reader.read_long()
trade_id = reader.read_long()
product_id = reader.read_string()
side = 'buy' if reader.read_short() == 0 else 'sell'
size = reader.read_double()
price = reader.read_double()
record = {
'time': datetime.datetime.fromtimestamp(time),
'sequence': sequence,
'trade_id': trade_id,
'product_id': product_id,
'side': side,
'size': size,
'price': price
}
records.append(record)
if len(records) > 0:
logger.info(
f'uploading journaled {exchange}/{symbol} ticks to Behemoth for UTC date {str(upload_date)}'
)
df = pd.DataFrame(records)
df.set_index('time', inplace=True)
logger.info(f'extracted {len(df)} {symbol} trade records')
tickstore = LocalTickstore(
Path(Path(f'{behemoth_path}/db/{db}')), 'time')
tickstore.insert(symbol, BiTimestamp(upload_date), df)
tickstore.close()
logger.info(f'inserted {len(df)} {symbol} records')
else:
logger.info(
f'zero {exchange}/{symbol} ticks for UTC date {str(upload_date)}'
)
tickstore = LocalTickstore(
Path(Path(f'{behemoth_path}/db/{db}')), 'time')
tickstore.close()
except NoSuchJournalException:
logger.error(f'missing journal file: {path}')
def backfill_coinbasepro(api_key: str, api_secret: str, api_passphrase: str):
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
exch_service = ExchangeEntityService(cur, type_code_cache,
instrument_cache)
auth_client = coinbasepro.AuthenticatedClient(key=api_key,
secret=api_secret,
passphrase=api_passphrase)
# Coinbase Pro has a notion of account per currency for tracking balances, so we want to pull
# out what it calls the profile, which is the parent exchange account
profile_set = set()
for account in auth_client.get_accounts():
profile_set.add(account['profile_id'])
exchange = type_code_cache.get_by_code(Exchange, "CoinbasePro")
account_by_profile_id = {}
for profile in profile_set:
account = exch_service.get_or_create_account(
ExchangeAccount(0, exchange, profile))
account_by_profile_id[profile] = account
# load up all the orders
for order in auth_client.get_orders(status=['done']):
order_uuid = order['id']
# market orders have no price
if 'price' in order:
price = order['price']
else:
price = None
# market orders that specify "funds" have no size
if 'size' in order:
size = order['size']
else:
size = order['filled_size']
exchange_account = account_by_profile_id[order['profile_id']]
instrument_type = type_code_cache.get_by_code(InstrumentType,
'CurrencyPair')
instrument = instrument_cache.get_or_create_instrument(
order['product_id'], instrument_type)
exchange_instrument = instrument_cache.get_or_create_exchange_instrument(
order['product_id'], instrument, exchange.get_type_code())
side = type_code_cache.get_by_code(Side, order['side'].capitalize())
if order['type'] is None:
order['type'] = 'Market'
order_type = type_code_cache.get_by_code(OrderType,
order['type'].capitalize())
if 'time_in_force' in order:
tif = type_code_cache.get_by_code(TimeInForce,
order['time_in_force'])
else:
tif = type_code_cache.get_by_code(TimeInForce, 'Day')
create_time = order['created_at']
order = ExchangeOrder(0, exchange, exchange_instrument, order_type,
exchange_account, side, tif, order_uuid, price,
size, create_time)
exch_service.get_or_create_exchange_order(order)
conn.commit()
# load up all the fills, linking back to the orders
for product in ['BTC-USD', 'ETH-BTC']:
for fill in auth_client.get_fills(product_id=product):
order_id = fill['order_id']
trade_id = fill['trade_id']
price = fill['price']
size = fill['size']
fees = fill['fee']
create_time = fill['created_at']
order = exch_service.get_entity_by_ak(
ExchangeOrder, (exchange.get_type_code(), order_id))
fill = ExchangeFill(0, price, size, fees, trade_id, create_time)
fill.set_order(order)
exch_service.get_or_create_exchange_fill(fill)
conn.commit()
def __init__(self, cur, type_code_cache: TypeCodeCache, instrument_cache: InstrumentCache):
self.cur = cur
self.type_code_cache = type_code_cache
self.instrument_cache = instrument_cache
self.trades = {}
cur.execute("SELECT eo.side_id, eo.exchange_instrument_id, ef.fill_price, ef.quantity, ef.fees, ef.create_time "
"FROM serenity.exchange_fill ef "
"INNER JOIN serenity.exchange_order eo on eo.exchange_order_id = ef.exchange_order_id "
"ORDER BY ef.create_time ASC ")
for row in cur.fetchall():
side = type_code_cache.get_by_id(Side, row[0])
instrument = instrument_cache.get_entity_by_id(ExchangeInstrument, row[1])
instrument_code = instrument.get_instrument().get_instrument_code()
if instrument_code not in self.trades:
self.trades[instrument_code] = {}
if side.get_type_code() not in self.trades[instrument_code]:
self.trades[instrument_code][side.get_type_code()] = []
trade_info = {
'px': row[2],
'qty': row[3],
'remaining': row[3],
'fee': row[4],
'ts': row[5],
}
self.trades[instrument_code][side.get_type_code()].append(trade_info)
# explode cross-currency trades into their component parts
usd_ccy = instrument_cache.get_entity_by_ak(Currency, 'USD')
for instrument_code in list(self.trades):
pair_codes = tuple(instrument_code.split('-'))
base_ccy = instrument_cache.get_entity_by_ak(Currency, pair_codes[0])
quote_ccy = instrument_cache.get_entity_by_ak(Currency, pair_codes[1])
if quote_ccy.get_currency_code() != 'USD':
long = "{}-{}".format(base_ccy.get_currency_code(), usd_ccy.get_currency_code())
short = "{}-{}".format(quote_ccy.get_currency_code(), usd_ccy.get_currency_code())
if long not in self.trades:
self.trades[long] = {'Buy': [], 'Sell': []}
if short not in self.trades:
self.trades[short] = {'Buy': [], 'Sell': []}
# for each sell, buy short instrument and sell long instrument
for sell in self.trades[instrument_code]['Sell']:
self.trades[short]['Buy'].append({
'px': self.lookup_instrument_mark(short, sell['ts']),
'qty': sell['qty'] * sell['px'],
'remaining': sell['qty'] * sell['px'],
'fee': 0.0,
'ts': sell['ts']
})
self.trades[long]['Sell'].append({
'px': self.lookup_instrument_mark(long, sell['ts']),
'qty': sell['qty'],
'remaining': sell['qty'],
'fee': 0.0,
'ts': sell['ts']
})
# for each buy, buy long instrument and sell short instrument
for buy in self.trades[instrument_code]['Buy']:
self.trades[long]['Buy'].append({
'px': self.lookup_instrument_mark(long, buy['ts']),
'qty': buy['qty'],
'remaining': buy['qty'],
'fee': 0.0,
'ts': buy['ts']
})
self.trades[short]['Sell'].append({
'px': self.lookup_instrument_mark(short, buy['ts']),
'qty': buy['qty'] * buy['px'],
'remaining': buy['qty'] * buy['px'],
'fee': 0.0,
'ts': buy['ts']
})
# re-sort buys and sells by ts
for instrument_code in list(self.trades):
self.trades[instrument_code]['Buy'] = sorted(self.trades[instrument_code]['Buy'], key=lambda k: k['ts'])
self.trades[instrument_code]['Sell'] = sorted(self.trades[instrument_code]['Sell'], key=lambda k: k['ts'])
def backfill_gemini(gemini_api_key: str, gemini_api_secret: str):
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
exch_service = ExchangeEntityService(cur, type_code_cache,
instrument_cache)
client = gemini.PrivateClient(gemini_api_key, gemini_api_secret)
for exchange_symbol in ('BTCUSD', 'ETHBTC', 'ZECBTC'):
instrument_symbol = exchange_symbol[0:3] + '-' + exchange_symbol[3:]
instrument_type = type_code_cache.get_by_code(InstrumentType,
'CurrencyPair')
instrument = instrument_cache.get_or_create_instrument(
instrument_symbol, instrument_type)
exchange_instrument = instrument_cache.get_or_create_exchange_instrument(
exchange_symbol.lower(), instrument, 'Gemini')
conn.commit()
exchange = type_code_cache.get_by_code(Exchange, 'Gemini')
for trade in client.get_past_trades(exchange_symbol):
fill_price = trade['price']
quantity = trade['amount']
fees = trade['fee_amount']
side = type_code_cache.get_by_code(Side, trade['type'])
trade_id = trade['tid']
order_uuid = trade['order_id']
create_time_ms = trade['timestampms']
create_time = datetime.utcfromtimestamp(create_time_ms // 1000).\
replace(microsecond=create_time_ms % 1000 * 1000)
# because we cannot get historical exchange orders past 7 days, we need to synthesize limit orders
exchange_account = ExchangeAccount(0, exchange, 'default')
exchange_account = exch_service.get_or_create_account(
exchange_account)
order_type = type_code_cache.get_by_code(OrderType, 'Limit')
tif = type_code_cache.get_by_code(TimeInForce, 'GTC')
order = ExchangeOrder(0, exchange, exchange_instrument, order_type,
exchange_account, side, tif, order_uuid,
fill_price, quantity, create_time)
order = exch_service.get_or_create_exchange_order(order)
conn.commit()
# create the fills and insert, linking back to the synthetic order
fill = ExchangeFill(0, fill_price, quantity, fees, trade_id,
create_time)
fill.set_order(order)
exch_service.get_or_create_exchange_fill(fill)
conn.commit()
for transfer in client.api_query('/v1/transfers', {}):
transfer_type = type_code_cache.get_by_code(ExchangeTransferType,
transfer['type'])
transfer_method = type_code_cache.get_by_code(ExchangeTransferMethod,
"Blockchain")
currency = instrument_cache.get_or_create_currency(
transfer['currency'])
quantity = transfer['amount']
transfer_ref = transfer['txHash']
transfer_time_ms = transfer['timestampms']
transfer_time = datetime.utcfromtimestamp(transfer_time_ms // 1000). \
replace(microsecond=transfer_time_ms % 1000 * 1000)
transfer = ExchangeTransfer(0, exchange, transfer_type,
transfer_method, currency, quantity,
transfer_ref, transfer_time)
exch_service.get_or_create_exchange_transfer(transfer)
conn.commit()