def backfill_coinbase():
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
exch_service = ExchangeEntityService(cur, type_code_cache,
instrument_cache)
# create a default account for Coinbase
exchange = type_code_cache.get_by_code(Exchange, 'Coinbase')
account = exch_service.get_or_create_account(
ExchangeAccount(0, exchange, 'default'))
# create a BTC-USD symbol for Coinbase
product_id = 'BTC-USD'
instrument_type = type_code_cache.get_by_code(InstrumentType,
'CurrencyPair')
instrument = instrument_cache.get_or_create_instrument(
product_id, instrument_type)
exchange_instrument = instrument_cache.get_or_create_exchange_instrument(
product_id, instrument, 'Coinbase')
# synthesize market orders and fills for Coinbase purchases
side = type_code_cache.get_by_code(Side, 'Buy')
order_type = type_code_cache.get_by_code(OrderType, 'Market')
tif = type_code_cache.get_by_code(TimeInForce, 'Day')
# noinspection DuplicatedCode
def create_exchange_order_and_fill(price, quantity, fees, order_id,
create_time):
order = ExchangeOrder(0, exchange, exchange_instrument, order_type,
account, side, tif, order_id, price, quantity,
create_time)
order = exch_service.get_or_create_exchange_order(order)
conn.commit()
fill = ExchangeFill(0, price, quantity, fees, order_id, create_time)
fill.set_order(order)
exch_service.get_or_create_exchange_fill(fill)
conn.commit()
# investment purchase
create_exchange_order_and_fill(265.39, 1.13, 2.9993, 1,
datetime(2015, 1, 26))
# residual left in account after payment by Bitcoin
create_exchange_order_and_fill(238.47, 2.1 - 1.68136, 5.013, 2,
datetime(2015, 2, 13))
# investment purchase
create_exchange_order_and_fill(249.05, 0.5, 1.25, 3, datetime(2015, 2, 14))
def backfill_coinbasepro(api_key: str, api_secret: str, api_passphrase: str):
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
exch_service = ExchangeEntityService(cur, type_code_cache,
instrument_cache)
auth_client = coinbasepro.AuthenticatedClient(key=api_key,
secret=api_secret,
passphrase=api_passphrase)
# Coinbase Pro has a notion of account per currency for tracking balances, so we want to pull
# out what it calls the profile, which is the parent exchange account
profile_set = set()
for account in auth_client.get_accounts():
profile_set.add(account['profile_id'])
exchange = type_code_cache.get_by_code(Exchange, "CoinbasePro")
account_by_profile_id = {}
for profile in profile_set:
account = exch_service.get_or_create_account(
ExchangeAccount(0, exchange, profile))
account_by_profile_id[profile] = account
# load up all the orders
for order in auth_client.get_orders(status=['done']):
order_uuid = order['id']
# market orders have no price
if 'price' in order:
price = order['price']
else:
price = None
# market orders that specify "funds" have no size
if 'size' in order:
size = order['size']
else:
size = order['filled_size']
exchange_account = account_by_profile_id[order['profile_id']]
instrument_type = type_code_cache.get_by_code(InstrumentType,
'CurrencyPair')
instrument = instrument_cache.get_or_create_instrument(
order['product_id'], instrument_type)
exchange_instrument = instrument_cache.get_or_create_exchange_instrument(
order['product_id'], instrument, exchange.get_type_code())
side = type_code_cache.get_by_code(Side, order['side'].capitalize())
if order['type'] is None:
order['type'] = 'Market'
order_type = type_code_cache.get_by_code(OrderType,
order['type'].capitalize())
if 'time_in_force' in order:
tif = type_code_cache.get_by_code(TimeInForce,
order['time_in_force'])
else:
tif = type_code_cache.get_by_code(TimeInForce, 'Day')
create_time = order['created_at']
order = ExchangeOrder(0, exchange, exchange_instrument, order_type,
exchange_account, side, tif, order_uuid, price,
size, create_time)
exch_service.get_or_create_exchange_order(order)
conn.commit()
# load up all the fills, linking back to the orders
for product in ['BTC-USD', 'ETH-BTC']:
for fill in auth_client.get_fills(product_id=product):
order_id = fill['order_id']
trade_id = fill['trade_id']
price = fill['price']
size = fill['size']
fees = fill['fee']
create_time = fill['created_at']
order = exch_service.get_entity_by_ak(
ExchangeOrder, (exchange.get_type_code(), order_id))
fill = ExchangeFill(0, price, size, fees, trade_id, create_time)
fill.set_order(order)
exch_service.get_or_create_exchange_fill(fill)
conn.commit()
def backfill_gemini(gemini_api_key: str, gemini_api_secret: str):
conn = connect_serenity_db()
cur = conn.cursor()
type_code_cache = TypeCodeCache(cur)
instrument_cache = InstrumentCache(cur, type_code_cache)
exch_service = ExchangeEntityService(cur, type_code_cache,
instrument_cache)
client = gemini.PrivateClient(gemini_api_key, gemini_api_secret)
for exchange_symbol in ('BTCUSD', 'ETHBTC', 'ZECBTC'):
instrument_symbol = exchange_symbol[0:3] + '-' + exchange_symbol[3:]
instrument_type = type_code_cache.get_by_code(InstrumentType,
'CurrencyPair')
instrument = instrument_cache.get_or_create_instrument(
instrument_symbol, instrument_type)
exchange_instrument = instrument_cache.get_or_create_exchange_instrument(
exchange_symbol.lower(), instrument, 'Gemini')
conn.commit()
exchange = type_code_cache.get_by_code(Exchange, 'Gemini')
for trade in client.get_past_trades(exchange_symbol):
fill_price = trade['price']
quantity = trade['amount']
fees = trade['fee_amount']
side = type_code_cache.get_by_code(Side, trade['type'])
trade_id = trade['tid']
order_uuid = trade['order_id']
create_time_ms = trade['timestampms']
create_time = datetime.utcfromtimestamp(create_time_ms // 1000).\
replace(microsecond=create_time_ms % 1000 * 1000)
# because we cannot get historical exchange orders past 7 days, we need to synthesize limit orders
exchange_account = ExchangeAccount(0, exchange, 'default')
exchange_account = exch_service.get_or_create_account(
exchange_account)
order_type = type_code_cache.get_by_code(OrderType, 'Limit')
tif = type_code_cache.get_by_code(TimeInForce, 'GTC')
order = ExchangeOrder(0, exchange, exchange_instrument, order_type,
exchange_account, side, tif, order_uuid,
fill_price, quantity, create_time)
order = exch_service.get_or_create_exchange_order(order)
conn.commit()
# create the fills and insert, linking back to the synthetic order
fill = ExchangeFill(0, fill_price, quantity, fees, trade_id,
create_time)
fill.set_order(order)
exch_service.get_or_create_exchange_fill(fill)
conn.commit()
for transfer in client.api_query('/v1/transfers', {}):
transfer_type = type_code_cache.get_by_code(ExchangeTransferType,
transfer['type'])
transfer_method = type_code_cache.get_by_code(ExchangeTransferMethod,
"Blockchain")
currency = instrument_cache.get_or_create_currency(
transfer['currency'])
quantity = transfer['amount']
transfer_ref = transfer['txHash']
transfer_time_ms = transfer['timestampms']
transfer_time = datetime.utcfromtimestamp(transfer_time_ms // 1000). \
replace(microsecond=transfer_time_ms % 1000 * 1000)
transfer = ExchangeTransfer(0, exchange, transfer_type,
transfer_method, currency, quantity,
transfer_ref, transfer_time)
exch_service.get_or_create_exchange_transfer(transfer)
conn.commit()