def collect_order_book():
gdax_order_book = gdax_client.get_order_book(ticker='eth', level=2)
gob_model = OrderBookModel.build(gdax_order_book)
gob_model.db_save(es)
cex_order_book = cex_client.get_order_book(ticker='eth', level=2)
cob_model = OrderBookModel.build(cex_order_book)
cob_model.db_save(es)
timestamp = gob_model.js['timestamp__long']
logger.info("saved data: {0}, {1}".format(str(timestamp),
epoch.to_str(timestamp)))
def get_order_book_data(self):
try:
gdax_order_book = self.gdax_public_client.get_product_order_book('ETH-USD', 2)
gob_model = OrderBookModel.build(gdax_order_book)
gob_model.db_save(es)
cex_order_book = self.cex_public_client.get_product_order_book('ETH-USD')
cob_model = OrderBookModel.build(cex_order_book)
cob_model.db_save(es)
logger.info("saved data: gdx:{0} | cex:{1}".format(gob_model.uid, cob_model.uid))
except Exception:
sentry_client.captureException()
tb = traceback.format_exc()
logger.error(tb)
def get_ds(strategy_run_id, check_window):
window = tuple(epoch.to_long(x) for x in check_window)
query = {
"size": 1000,
"sort": [{
"timestamp__long": {
"order": "asc"
}
}],
"query": {
"bool": {
"must": [{
"match": {
"strategy_run_id.raw": strategy_run_id
}
}, {
"range": {
"timestamp__long": {
"gte": window[0],
"lte": window[1]
}
}
}]
}
}
}
logger.info(pretty_json(query))
hits = es.search('audit_trading', 'data', query)['hits']['hits']
# Data vectors
x = []
y_total_usd = []
y_gdax_usd = []
y_cex_usd = []
for hit in hits:
hit = hit['_source']
x.append(hit['timestamp__long'])
y_total_usd.append(hit['total_usd__num'])
y_gdax_usd.append(hit['gdax_account']['usd__num'])
y_cex_usd.append(hit['cex_account']['usd__num'])
# Data series
x_index = pd.to_datetime(x, unit='ms')
ds1 = pd.Series(index=x_index, data=y_total_usd)
ds2 = pd.Series(index=x_index, data=y_gdax_usd)
ds3 = pd.Series(index=x_index, data=y_cex_usd)
return ds1, ds2, ds3
def get_ds_trading_result(check_window, check_interval, amount, holding_period, threshold_delta,
gdax_trading_account, cex_trading_account):
"""
data series in dates that:
1. show trading results
"""
window = tuple(epoch.to_long(x) for x in check_window)
interval = MILLIS_IN_MINUTE * check_interval
x = []
y1 = []
y2 = []
# Use the strategy to calculate deltas
strategy001 = Strat1(None, None, [gdax_trading_account, cex_trading_account])
n = (window[1] - window[0]) / interval
timestamp = window[0]
for i in range(n):
withdraw_signal = strategy001.get_signal__withdraw_delta(timestamp)
withdraw_delta = withdraw_signal['withdraw_delta']
if withdraw_delta > threshold_delta:
result = trade_result(amount, holding_period, timestamp, gdax_trading_account, cex_trading_account)
x.append(timestamp)
y1.append(withdraw_delta)
y2.append(result)
# print i, timestamp, epoch.to_str(timestamp), withdraw_delta, result
# next timestamp
timestamp += interval
# checking progress
if i % 50 == 0:
logger.info("[{}] timestamp:{}|date:{}".format(str(i), str(timestamp), epoch.to_str(timestamp)))
x_index = pd.to_datetime(x, unit='ms')
ds1 = pd.Series(index=x_index, data=y1)
ds2 = pd.Series(index=x_index, data=y2)
logger.info(check_window)
logger.info(window)
logger.info(ds1.head())
logger.info(ds1.head())
return ds1, ds2
def store_error_info(obj, exception):
"""
Parameters
----------
file_path: str
A string representing the file path
Returns
-------
core.models.Concept
"""
traceback_stack_msg = traceback.format_exc()
logger.info(traceback_stack_msg)
obj['error_type'] = type(exception).__name__
obj['error_msg'] = exception.message
obj['error_traceback_list'] = traceback_stack_msg.split('\n')
def get_two_deltas(check_window, check_interval):
window = tuple(epoch.to_long(x) for x in check_window)
interval = MILLIS_IN_MINUTE * check_interval
x = []
y1 = []
y2 = []
# Use the strategy to calculate deltas
gdax_t_account = BacktestingTradingAccount('backtesting_gdx_001', 'gdax')
cex_t_account = BacktestingTradingAccount('backtesting_cex_001', 'cex')
strategy001 = Strat1(None, None, [gdax_t_account, cex_t_account])
n = (window[1] - window[0]) / interval
timestamp = window[0]
for i in range(n):
withdraw_signal = strategy001.get_signal__withdraw_delta(timestamp)
deposit_signal = strategy001.get_signal__deposit_delta(timestamp)
withdraw_delta = withdraw_signal['withdraw_delta']
deposit_delta = deposit_signal['deposit_delta']
x.append(timestamp)
y1.append(withdraw_delta)
y2.append(deposit_delta)
# print i, timestamp, epoch.to_str(timestamp)
# next timestamp
timestamp += interval
# checking progress
if i % 50 == 0:
logger.info("timestamp:{0}|date:{1}".format(str(timestamp), epoch.to_str(timestamp)))
x_index = pd.to_datetime(x, unit='ms')
ds1 = pd.Series(index=x_index, data=y1)
ds2 = pd.Series(index=x_index, data=y2)
logger.info(check_window)
logger.info(window)
logger.info(ds1.head())
logger.info(ds1.head())
return ds1, ds2
def get_order_book(self, product_id=None, ticker=None, timestamp=-1):
"""
Search from timstamp, and work backward until one is found.
:return: OrderbookModel
"""
if timestamp == -1:
timestamp = epoch.current_milli_time()
search_window = 11000 # 11 seconds
search_window = 15 * 60000 # 15 minutes
search_window = 1000 * 60000 # 1000 minutes
(t0, t1) = timestamp - search_window, timestamp + 500
query = {
"size": 1,
"sort": [
{
"timestamp__long": {
"order": "desc"
}
}
],
"query": {
"bool": {
"must": [
{
"range": {
"timestamp__long": {
"gte": t0,
"lte": t1
}
}
},
{
"match": {
"exchange.raw": self.exh
}
},
{
"match": {
"product.raw": 'eth-usd'
}
}
]
}
}
}
params = default_es_get_params()
result = es.search("order_book", "data", query, params=params)
# validation
if result['hits']['total'] == 0:
logger.info(pretty_json(query))
logger.info(timestamp)
logger.info(epoch.to_str(timestamp))
raise RuntimeError('Cannot find orderbook in backtesting')
ob_js = result['hits']['hits'][0]['_source']
ob = OrderBookModel.build(ob_js)
return ob
def test__03(self):
"""
sell stopping price
"""
ob_js = {
"uid":
"71e689f5d4884dddb17f3f3b660b40f1",
"created__long":
1503543603464,
"modified__long":
1503543603468,
"exchange":
"gdax",
"product":
"eth-usd",
"bids": [{
"price__num": 316.17,
"size__num": 2.39426145,
"num_orders__int": 1.0
}, {
"price__num": 316.01,
"size__num": 0.03269,
"num_orders__int": 1.0
}, {
"price__num": 316.0,
"size__num": 22.6947,
"num_orders__int": 6.0
}]
}
ob = OrderBookModel.parse(ob_js)
shares = 10
stopping_price = helpers.compute_sell_stopping_price(shares, ob)
logger.info(stopping_price * 0.999)
lowerbound = helpers.compute_sell_lowerbound(shares, ob)
logger.info(ob)
logger.info(stopping_price)
logger.info(lowerbound)
def test__run_main(self):
trading_account = LiveTradingAccount('gdax', 'gdax')
account = trading_account.sync_account_with_exh()
logger.info('before')
logger.info(account)
# account_02 = trading_account.place_limit_order('buy', 'eth', 100.0, 0.01)
account_02 = trading_account.place_limit_order('sell', 'eth', 500.0,
0.01)
# ob = trading_account.get_order_book('eth')
# shares = 20.0
logger.info('before')
logger.info('-' * 30)
logger.info(account)
logger.info('after')
logger.info('-' * 30)
logger.info(account_02)
assert True
from arb import logger, es
from arb.core.exh.backtest.accounts import MockTradingAccount
from arb.core.models import AccountModel
from arb.strat.runner import StrategyRunner
from arb.strat.strat1 import Strat1
from arb.utils import epoch
if __name__ == '__main__':
logger.info("Starting a mock live strategy...")
strategy_desc = 'mock live 00, 2007/09/23, Saturday'
strategy_running_id = 'server_mock_live__004'
gdax_account_name = strategy_running_id + '__gdax'
cex_account_name = strategy_running_id + '__cex'
start_time = epoch.current_milli_time()
execution_window = [start_time, start_time + 7884000000] # three months
execution_interval = 5 * 60 * 1000 # every five minutes
# execution_window = [start_time, start_time + 60000] # testing, 30 seconds
# execution_interval = 10000 # testing, every 10 seconds
# set accounts
gdax_acc_js = {
"uid": gdax_account_name,
"timestamp__long": start_time,
"exchange": "gdax",
"country": "usa",
"usd__num": 20000.0,
"eth__num": 3.0,
"btc__num": 0.0
def log_repeat():
threading.Timer(3, log_repeat).start()
logger.info('logging: ' + epoch.current_time())
def start(self, fail_fast=True):
logger.info('Starting strategy: {0}'.format(
str(self.strategy.strategy_name)))
logger.info("Strategy withdraw threshold: " +
str(self.strategy.THRESHOLD_WITHDRAW_DELTA))
logger.info("Strategy deposit threshold: " +
str(self.strategy.THRESHOLD_DEPOSIT_DELTA))
# How long? How often?
length = (self.execution_window[1] - self.execution_window[0]
) # In millisecond
interval = self.execution_interval # In millisecond
# length = 86400000 # one day
# interval = 600 # 5 minutes
counter_limit = length / interval
timestamp = self.execution_window[0]
logger.info('====================================')
logger.info('Starting time : {0}'.format(epoch.to_str(timestamp)))
logger.info('Interval in seconds: {0}'.format(str(interval / 1000)))
logger.info('====================================')
counter = 0
while counter <= counter_limit:
counter += 1
timestamp += interval
snap_t = self.strategy.trading_acc1.get_snapping_timestamp(
timestamp)
self.snap(snap_t, fail_fast=fail_fast) # main execution path
# logging info line
execution_timestamp = '-1' if timestamp == -1 else epoch.to_str(
timestamp)
info_line = '{0} Runner snapping, current time: {1} | execution timestamp: {2}' \
.format(str(counter), epoch.current_time(), str(execution_timestamp))
logger.info(info_line)
self.strategy.trading_acc1.sleep(interval /
1000) # Sleeping in seconds
logger.info('Finished strategy runner : {0}'.format(
epoch.current_time()))
def get_results_for_a_mock_strategy(check_window, check_interval, amount, holding_period, threshold_delta,
gdax_trading_account, cex_trading_account):
"""
keep track of how we we are doing with one trade at a time
"""
window = tuple(epoch.to_long(x) for x in check_window)
interval = MILLIS_IN_MINUTE * check_interval
x = []
y1 = []
y2 = []
# Use the strategy to calculate deltas
strategy001 = Strat1(None, None, [gdax_trading_account, cex_trading_account])
n = (window[1] - window[0]) / interval
ONE_DAY_IN_MINUTES =1440
cash = amount
eth = 0.0
waiting_liquidate_ticks = 0
waiting_capital_ticks = ONE_DAY_IN_MINUTES
# def signal__has_eth():
# return eth > 0.0
timestamp = window[0]
for i in range(n):
waiting_liquidate_ticks += check_interval
waiting_capital_ticks += check_interval
withdraw_signal = strategy001.get_signal__withdraw_delta(timestamp)
withdraw_delta = withdraw_signal['withdraw_delta']
# handling gdax
if withdraw_delta >= threshold_delta and eth == 0.0 and waiting_capital_ticks >= ONE_DAY_IN_MINUTES:
gdax_ob = gdax_trading_account.get_order_book(ticker='eth', timestamp=timestamp)
shares = helpers.compute_buy(amount, gdax_ob)
usd_used = helpers.compute_usd_spent(shares, gdax_ob)
# accounting
cash = cash - usd_used
eth = eth + shares * (1 - 0.003) # Including fees
waiting_liquidate_ticks = 0
waiting_capital_ticks = 0
# x.append(timestamp)
# y1.append(cash)
# y2.append(eth)
if eth > 0.0 and waiting_liquidate_ticks >= holding_period:
cex_ob = cex_trading_account.get_order_book(ticker='eth', timestamp=timestamp)
shares = eth
usd_gotten = helpers.compute_usd_made(shares, cex_ob) * (1 - 0.002) # Including fees
# accounting
cash = cash + usd_gotten
eth = 0
x.append(timestamp)
y1.append(cash)
y2.append(eth)
print i, epoch.to_str(timestamp)
print "cash: {} | eth: {}".format(str(cash), str(eth))
print "capital tick: {} | liquidate tick: {}".format(waiting_capital_ticks, waiting_liquidate_ticks)
# next timestamp
timestamp += interval
# checking progress
# print i, epoch.to_str(timestamp)
# print "cash: {} | eth: {}".format(str(cash), str(eth))
# print "capital tick: {} | liquidate tick: {}".format(waiting_capital_ticks, waiting_liquidate_ticks)
if i % 50 == 0:
print i, epoch.to_str(timestamp)
print "cash: {} | eth: {}".format(str(cash), str(eth))
print "capital tick: {} | liquidate tick: {}".format(waiting_capital_ticks, waiting_liquidate_ticks)
x_index = pd.to_datetime(x, unit='ms')
ds1 = pd.Series(index=x_index, data=y1)
ds2 = pd.Series(index=x_index, data=y2)
logger.info(check_window)
logger.info(window)
logger.info(ds1.head())
return ds1, ds2
gdax_order_book = gdax_client.get_order_book(ticker='eth', level=2)
gob_model = OrderBookModel.build(gdax_order_book)
gob_model.db_save(es)
cex_order_book = cex_client.get_order_book(ticker='eth', level=2)
cob_model = OrderBookModel.build(cex_order_book)
cob_model.db_save(es)
timestamp = gob_model.js['timestamp__long']
logger.info("saved data: {0}, {1}".format(str(timestamp),
epoch.to_str(timestamp)))
def run():
while True:
try:
collect_order_book()
except Exception as e:
sentry_client.captureException()
tb = traceback.format_exc()
logger.error(tb)
time.sleep(delay)
if __name__ == '__main__':
logger.info('-' * 50)
logger.info("Starting background data collections...")
logger.info('-' * 50)
run()
def market_snap(self, timestamp=-1):
"""
Main logic about exactly what to do for each market interaction
"""
# refresh account states
self.trading_acc1.sync_account_with_exh()
self.trading_acc2.sync_account_with_exh()
signal__arbitrage_delta = self.get_signal__arbitrage_delta()
signal__gdax_has_usd = self.get_signal__gdax_has_usd()
signal__gdax_has_eth = self.get_signal__gdax_has_eth()
signal__cex_has_eth = self.get_signal__cex_has_eth()
def mk_audit_js():
gdax_account = self.trading_acc1.get_account()
cex_account = self.trading_acc2.get_account()
transaction_t = epoch.current_milli_time(
) if timestamp == -1 else timestamp
audit_js = OrderedDict()
audit_js['strategy_run_id'] = self.run_id
audit_js['timestamp'] = epoch.to_str(transaction_t)
audit_js['timestamp__long'] = transaction_t
audit_js['ticker'] = self.ticker
audit_js['strategy_info'] = self._strategy_info
audit_js['signal'] = OrderedDict()
audit_js['signal']['signal__gdax_has_usd'] = signal__gdax_has_usd
audit_js['signal']['signal__gdax_has_eth'] = signal__gdax_has_eth
audit_js['signal']['signal__cex_has_eth'] = signal__cex_has_eth
audit_js['signal'][
'signal__arbitrage_delta'] = signal__arbitrage_delta
audit_js['total_usd__num'] = gdax_account.js[
'usd__num'] + cex_account.js['usd__num']
audit_js['total_eth__num'] = gdax_account.js[
'eth__num'] + cex_account.js['eth__num']
audit_js['gdax_account'] = gdax_account.js
audit_js['cex_account'] = cex_account.js
return audit_js
snap_again = False # Only repeat if we have an gdax buy action
if signal__gdax_has_usd['signal'] and signal__arbitrage_delta['signal']:
exec_context = self.exec_gdax_buy(timestamp)
snap_again = True
# Audit
audit_js = mk_audit_js()
audit_js['action'] = exec_context
audit = AuditTradeModel.build(audit_js)
logger.info('-----Executed GDAX Buy-----')
logger.info(audit)
logger.info('---------------------------')
audit.db_save(es)
if signal__gdax_has_eth['signal']:
exec_context = self.exec_eth_transfer()
# Audit
audit_js = mk_audit_js()
audit_js['action'] = exec_context
audit = AuditTradeModel.build(audit_js)
logger.info('-----Executed ETH TRANSFER-----')
logger.info(audit)
logger.info('---------------------------')
audit.db_save(es)
if signal__cex_has_eth['signal']:
exec_context = self.exec_cex_sell(timestamp)
# Audit
audit_js = mk_audit_js()
audit_js['action'] = exec_context
audit = AuditTradeModel.build(audit_js)
logger.info('-----Executed CEX Sell-----')
logger.info(audit)
logger.info('---------------------------')
audit.db_save(es)
# Extra logging
audit_js = mk_audit_js()
logger.info('post-snapping states: \n' +
json.dumps(audit_js, indent=2))
return snap_again