def step(self, action):
assert self.action_space.contains(action)
try:
self.orderbook.append(self.data_generator.next())
except StopIteration:
done = True
best_bid = Orderbook.getBestBid()
best_ask = Orderbook.getBestAsk()
next_bid = Orderbook.getNextBestBid()
next_ask = Orderbook.getNExtBestAsk()
# Buy
if action == 0:
self.inventory += 1
# Sell
elif action == 1:
self.inventory -= 1
if self.inventory > 0:
start = best_ask
end = next_bid
elif self.inventory <= 0:
start = best_bid
end = next_ask
reward = (end / start - 1) * self.inventory
self.state += 1
done = False
return reward, done, {}
def setUp(self):
'''
setUp creates the Orderbook instance and a set of orders
'''
self.ex1 = Orderbook(5)
self.q0_buy = {'order_id': 't1_0', 'timestamp': 1, 'type': 'add', 'quantity': 1, 'side': 'buy',
'price': 50}
self.q1_buy = {'order_id': 't1_1', 'timestamp': 2, 'type': 'add', 'quantity': 1, 'side': 'buy',
'price': 50}
self.q2_buy = {'order_id': 't1_2', 'timestamp': 3, 'type': 'add', 'quantity': 1, 'side': 'buy',
'price': 50}
self.q3_buy = {'order_id': 't10_1', 'timestamp': 4, 'type': 'add', 'quantity': 3, 'side': 'buy',
'price': 49}
self.q4_buy = {'order_id': 't11_1', 'timestamp': 5, 'type': 'add', 'quantity': 3, 'side': 'buy',
'price': 47}
self.q5_buy = {'order_id': 't12_1', 'timestamp': 6, 'type': 'add', 'quantity': 3, 'side': 'buy',
'price': 47}
self.q6_buy = {'order_id': 't13_1', 'timestamp': 7, 'type': 'add', 'quantity': 3, 'side': 'buy',
'price': 47}
self.q0_sell = {'order_id': 't1_5', 'timestamp': 1, 'type': 'add', 'quantity': 1, 'side': 'sell',
'price': 52}
self.q1_sell = {'order_id': 't1_3', 'timestamp': 2, 'type': 'add', 'quantity': 1, 'side': 'sell',
'price': 52}
self.q2_sell = {'order_id': 't1_4', 'timestamp': 3, 'type': 'add', 'quantity': 1, 'side': 'sell',
'price': 52}
self.q3_sell = {'order_id': 't10_2', 'timestamp': 4, 'type': 'add', 'quantity': 3, 'side': 'sell',
'price': 53}
self.q4_sell = {'order_id': 't11_2', 'timestamp': 5, 'type': 'add', 'quantity': 3, 'side': 'sell',
'price': 55}
self.q5_sell = {'order_id': 't12_2', 'timestamp': 6, 'type': 'add', 'quantity': 3, 'side': 'sell',
'price': 53}
self.q6_sell = {'order_id': 't13_2', 'timestamp': 7, 'type': 'add', 'quantity': 3, 'side': 'sell',
'price': 55}
def load_orderbooks():
summary_file = relative_path('static', 'summary.json')
with open(summary_file, 'r') as f:
summaries = json.load(f)
downloadable = [item_summary for item_summary in summaries.values() if ITEM_FILTER(item_summary)]
orderbooks = [Orderbook(summaries, item['id']) for item in downloadable]
return [orderbook for orderbook in orderbooks if ORDERBOOK_FILTER(orderbook)]
def docMappings(self, connection, indexName=DEFAULT_INDEX): try: connection.indices.put_mapping(index=indexName, doc_type="orderbook", body=Orderbook().orderbookMapping) connection.indices.put_mapping(index=indexName, doc_type="ticker", body=Ticker().tickerMapping) connection.indices.put_mapping(index=indexName, doc_type="completed_trades", body=Trade().completedTradeMapping) connection.indices.put_mapping(index=indexName, doc_type="future_ticker", body=Ticker().futureTickerMapping) connection.indices.put_mapping(index=indexName, doc_type="future_price_index", body=FutureIndex().futurePriceIndexMapping) connection.indices.put_mapping(index=indexName, doc_type="kline_candles", body=KlineCandle().klineMapping) except: raise pass
def main():
# Load orderbook
orderbook = Orderbook()
orderbook.loadFromEvents('ob-1-small.tsv')
env = gym.make("ctc-executioner-v0")
env.configure(orderbook)
model = deepq.models.cnn_to_mlp( convs=[(1, 10, 20)], hiddens=[200])
act = deepq.learn(
env,
q_func=model,
lr=1e-4,
max_timesteps=100000,
buffer_size=5000,
exploration_fraction=0.1,
exploration_final_eps=0.1,
target_network_update_freq=1,
print_freq=10,
)
print("Saving model as ctc-executioner-v0.pkl")
act.save("ctc-executioner-v0.pkl")
class ItemSet(object):
def __init__(self, summary, set_description):
self.item_id = set_description['id']
self.etf_name = set_description['name']
self.orderbook = Orderbook(summary, self.item_id)
self.parts = [Orderbook(summary, part['id']) for part in set_description['parts']]
def sum_price_at(self, timestamp):
return sum(part.price_at(timestamp) for part in self.parts)
def etf_price_at(self, timestamp):
return self.orderbook.price_at(timestamp)
def sum_volume_at(self, timestamp):
return sum(part.volume_at(timestamp) for part in self.parts)
def min_volume_at(self, timestamp):
return min(part.volume_at(timestamp) for part in self.parts)
def etf_volume_at(self, timestamp):
return self.orderbook.volume_at(timestamp)
def sum_bid_at(self, timestamp):
return sum(part.bid_at(timestamp) for part in self.parts)
def etf_bid_at(self, timestamp):
return self.orderbook.bid_at(timestamp)
def sum_offer_at(self, timestamp):
return sum(part.offer_at(timestamp) for part in self.parts)
def etf_offer_at(self, timestamp):
return self.orderbook.offer_at(timestamp)
def __str__(self):
return 'ItemSet(id={}, parts={}, asset_name={})'.format(self.item_id, ', '.join(p.item_id for p in self.parts), self.etf_name)
def __repr__(self):
return self.__str__()
def __init__(self, n_days, n_steps_per_day, asset_config_path,
agent_config_path):
self.n_days = n_days
self.n_steps_per_day = n_steps_per_day
initial_shares_per_agent = 10
n_agents = AgentGenerator.n_agents(agent_config_path)
print(n_agents)
self.companies = CompanyGenerator.generate_companies(n_companies=10,
n_sectors=3)
self.tickers = [each.ticker for each in self.companies]
# self.tickers = 'EQNR DNO AKER MHG NRS LSG'.split(' ')
# self.companies = []
# for ticker in self.tickers:
# comp = Company(ticker=ticker,
# n_shares=initial_shares_per_agent * n_agents,
# cash=100 * initial_shares_per_agent * n_agents,
# yield_policy=0.10)
# self.companies.append(comp)
self.assets = [Stock(comp) for comp in self.companies]
self.agents = AgentGenerator.generate(agent_config_path,
self.assets,
verbose=True)
n_agents = len(self.agents)
for agent in self.agents:
for stock in self.assets:
agent.portfolio.assets[stock] = initial_shares_per_agent
self.cov_matrix = test.generate_corr_matrix(len(self.tickers), 2, 0.2)
self.weights = np.ones(len(self.tickers))
self.weights = self.weights / self.weights.size
self.book = Orderbook([], [])
# self.assets = ...
self.timestamp = Timestamp(0, 0)
self.market_portfolio = Portfolio.empty_portfolio(self.assets)
self.exchange = Exchange(self.agents, self.market_portfolio,
n_steps_per_day)
self.history = []
def subscribe_orderbook(self, symbol1, symbol2, depth=5):
ccy = "{}:{}".format(symbol1, symbol2)
self.ccy_order_books[ccy] = Orderbook(ccy, depth, self.logger)
self.logger.info("[WS] Subscribing to pair {}".format(ccy))
self.debug_init_time[ccy] = time.time()
if not self.is_connected:
self.start()
self.logger.warning("Not connected, trying to reconnect")
time.sleep(2)
oid = "{}_orderbook_{}{}".format(str(self.get_timestamp()), symbol1,
symbol2)
msg = json.dumps({
"e": "order-book-subscribe",
"data": {
"pair": [symbol1, symbol2],
"subscribe": True,
"depth": depth
},
"oid": oid
})
self.ws.send(msg)
import unittest
from action import Action
from orderbook import Orderbook
from order import Order
from order_type import OrderType
from order_side import OrderSide
orderbook = Orderbook(extraFeatures=False)
orderbook.loadFromFile('test_orderbook_10s.tsv')
class MatchEngineMock():
def matchOrder(self, order, seconds=None):
trades = []
qtyRemain = 0
index = 1
return trades, qtyRemain, index
class ActionMock(Action):
def __init__(self, a, runtime):
Action.__init__(self, a, runtime)
def getMatchEngine(self, orderbook):
return MatchEngineMock()
class ActionTest(unittest.TestCase):
def testRun(self):
a = 1
i = 1.0
def getOrderbook(self):
o = Orderbook()
o.addState
def reset_orderbooks(self):
for each in self.portfolio.get_stocks():
bid, ask = each.get_last_price(), each.get_last_price()
self.orderbooks[each] = Orderbook(bid, ask, [], [])
class TestOrderbook(unittest.TestCase):
'''
Attribute objects in the Orderbook class include:
order_history: list
_bid_book: dictionary
_bid_book_prices: sorted list
_ask_book: dictionary
_ask_book_prices: sorted list
confirm_modify_collector: list
confirm_trade_collector: list
sip_collector: list
trade_book: list
Each method impacts one or more of these attributes.
'''
def setUp(self):
'''
setUp creates the Orderbook instance and a set of orders
'''
self.ex1 = Orderbook()
self.q1_buy = {
'order_id': 't1_1',
'timestamp': 2,
'type': 'add',
'quantity': 1,
'side': 'buy',
'price': 50
}
self.q2_buy = {
'order_id': 't1_2',
'timestamp': 3,
'type': 'add',
'quantity': 1,
'side': 'buy',
'price': 50
}
self.q3_buy = {
'order_id': 't10_1',
'timestamp': 4,
'type': 'add',
'quantity': 3,
'side': 'buy',
'price': 49
}
self.q4_buy = {
'order_id': 't11_1',
'timestamp': 5,
'type': 'add',
'quantity': 3,
'side': 'buy',
'price': 47
}
self.q1_sell = {
'order_id': 't1_3',
'timestamp': 2,
'type': 'add',
'quantity': 1,
'side': 'sell',
'price': 52
}
self.q2_sell = {
'order_id': 't1_4',
'timestamp': 3,
'type': 'add',
'quantity': 1,
'side': 'sell',
'price': 52
}
self.q3_sell = {
'order_id': 't10_2',
'timestamp': 4,
'type': 'add',
'quantity': 3,
'side': 'sell',
'price': 53
}
self.q4_sell = {
'order_id': 't11_2',
'timestamp': 5,
'type': 'add',
'quantity': 3,
'side': 'sell',
'price': 55
}
def test_add_order_to_history(self):
'''
add_order_to_history() impacts the order_history list
'''
h1 = {
'order_id': 't1_5',
'timestamp': 4,
'type': 'add',
'quantity': 5,
'side': 'sell',
'price': 55
}
self.assertFalse(self.ex1.order_history)
h1['exid'] = 1
self.ex1._add_order_to_history(h1)
self.assertDictEqual(h1, self.ex1.order_history[0])
def test_add_order_to_book(self):
'''
add_order_to_book() impacts _bid_book and _bid_book_prices or _ask_book and _ask_book_prices
Add two buy orders, then two sell orders
'''
# 2 buy orders
self.assertFalse(self.ex1._bid_book_prices)
self.assertFalse(self.ex1._bid_book)
self.ex1.add_order_to_book(self.q1_buy)
self.assertTrue(50 in self.ex1._bid_book_prices)
self.assertTrue(50 in self.ex1._bid_book.keys())
self.assertEqual(self.ex1._bid_book[50]['num_orders'], 1)
self.assertEqual(self.ex1._bid_book[50]['size'], 1)
self.assertEqual(self.ex1._bid_book[50]['order_ids'][0],
self.q1_buy['order_id'])
self.assertDictEqual(
self.ex1._bid_book[50]['orders'][self.q1_buy['order_id']],
self.q1_buy)
self.ex1.add_order_to_book(self.q2_buy)
self.assertEqual(self.ex1._bid_book[50]['num_orders'], 2)
self.assertEqual(self.ex1._bid_book[50]['size'], 2)
self.assertEqual(self.ex1._bid_book[50]['order_ids'][1],
self.q2_buy['order_id'])
self.assertDictEqual(
self.ex1._bid_book[50]['orders'][self.q2_buy['order_id']],
self.q2_buy)
# 2 sell orders
self.assertFalse(self.ex1._ask_book_prices)
self.assertFalse(self.ex1._ask_book)
self.ex1.add_order_to_book(self.q1_sell)
self.assertTrue(52 in self.ex1._ask_book_prices)
self.assertTrue(52 in self.ex1._ask_book.keys())
self.assertEqual(self.ex1._ask_book[52]['num_orders'], 1)
self.assertEqual(self.ex1._ask_book[52]['size'], 1)
self.assertEqual(self.ex1._ask_book[52]['order_ids'][0],
self.q1_sell['order_id'])
self.assertDictEqual(
self.ex1._ask_book[52]['orders'][self.q1_sell['order_id']],
self.q1_sell)
self.ex1.add_order_to_book(self.q2_sell)
self.assertEqual(self.ex1._ask_book[52]['num_orders'], 2)
self.assertEqual(self.ex1._ask_book[52]['size'], 2)
self.assertEqual(self.ex1._ask_book[52]['order_ids'][1],
self.q2_sell['order_id'])
self.assertDictEqual(
self.ex1._ask_book[52]['orders'][self.q2_sell['order_id']],
self.q2_sell)
def test_remove_order(self):
'''
_remove_order() impacts _bid_book and _bid_book_prices or _ask_book and _ask_book_prices
Add two orders, remove the second order twice
'''
# buy orders
self.ex1.add_order_to_book(self.q1_buy)
self.ex1.add_order_to_book(self.q2_buy)
self.assertTrue(50 in self.ex1._bid_book_prices)
self.assertTrue(50 in self.ex1._bid_book.keys())
self.assertEqual(self.ex1._bid_book[50]['num_orders'], 2)
self.assertEqual(self.ex1._bid_book[50]['size'], 2)
self.assertEqual(len(self.ex1._bid_book[50]['order_ids']), 2)
# remove first order
self.ex1._remove_order('buy', 50, 't1_1')
self.assertEqual(self.ex1._bid_book[50]['num_orders'], 1)
self.assertEqual(self.ex1._bid_book[50]['size'], 1)
self.assertEqual(len(self.ex1._bid_book[50]['order_ids']), 1)
self.assertFalse('t1_1' in self.ex1._bid_book[50]['orders'].keys())
self.assertTrue(50 in self.ex1._bid_book_prices)
# remove second order
self.ex1._remove_order('buy', 50, 't1_2')
self.assertFalse(self.ex1._bid_book_prices)
self.assertEqual(self.ex1._bid_book[50]['num_orders'], 0)
self.assertEqual(self.ex1._bid_book[50]['size'], 0)
self.assertEqual(len(self.ex1._bid_book[50]['order_ids']), 0)
self.assertFalse('t1_2' in self.ex1._bid_book[50]['orders'].keys())
self.assertFalse(50 in self.ex1._bid_book_prices)
# remove second order again
self.ex1._remove_order('buy', 50, 't1_2')
self.assertFalse(self.ex1._bid_book_prices)
self.assertEqual(self.ex1._bid_book[50]['num_orders'], 0)
self.assertEqual(self.ex1._bid_book[50]['size'], 0)
self.assertEqual(len(self.ex1._bid_book[50]['order_ids']), 0)
self.assertFalse('t1_2' in self.ex1._bid_book[50]['orders'].keys())
# sell orders
self.ex1.add_order_to_book(self.q1_sell)
self.ex1.add_order_to_book(self.q2_sell)
self.assertTrue(52 in self.ex1._ask_book_prices)
self.assertTrue(52 in self.ex1._ask_book.keys())
self.assertEqual(self.ex1._ask_book[52]['num_orders'], 2)
self.assertEqual(self.ex1._ask_book[52]['size'], 2)
self.assertEqual(len(self.ex1._ask_book[52]['order_ids']), 2)
# remove first order
self.ex1._remove_order('sell', 52, 't1_3')
self.assertEqual(self.ex1._ask_book[52]['num_orders'], 1)
self.assertEqual(self.ex1._ask_book[52]['size'], 1)
self.assertEqual(len(self.ex1._ask_book[52]['order_ids']), 1)
self.assertFalse('t1_1' in self.ex1._ask_book[52]['orders'].keys())
self.assertTrue(52 in self.ex1._ask_book_prices)
# remove second order
self.ex1._remove_order('sell', 52, 't1_4')
self.assertFalse(self.ex1._ask_book_prices)
self.assertEqual(self.ex1._ask_book[52]['num_orders'], 0)
self.assertEqual(self.ex1._ask_book[52]['size'], 0)
self.assertEqual(len(self.ex1._ask_book[52]['order_ids']), 0)
self.assertFalse('t1_2' in self.ex1._ask_book[52]['orders'].keys())
self.assertFalse(52 in self.ex1._ask_book_prices)
# remove second order again
self.ex1._remove_order('sell', 52, 't1_4')
self.assertFalse(self.ex1._ask_book_prices)
self.assertEqual(self.ex1._ask_book[52]['num_orders'], 0)
self.assertEqual(self.ex1._ask_book[52]['size'], 0)
self.assertEqual(len(self.ex1._ask_book[52]['order_ids']), 0)
self.assertFalse('t1_2' in self.ex1._ask_book[52]['orders'].keys())
def test_modify_order(self):
'''
_modify_order() primarily impacts _bid_book or _ask_book
_modify_order() could impact _bid_book_prices or _ask_book_prices if the order results
in removing the full quantity with a call to _remove_order()
Add 1 order, remove partial, then remainder
'''
# Buy order
q1 = {
'order_id': 't1_1',
'timestamp': 5,
'type': 'add',
'quantity': 2,
'side': 'buy',
'price': 50
}
self.ex1.add_order_to_book(q1)
self.assertEqual(self.ex1._bid_book[50]['size'], 2)
# remove 1
self.ex1._modify_order('buy', 1, 't1_1', 50)
self.assertEqual(self.ex1._bid_book[50]['size'], 1)
self.assertEqual(self.ex1._bid_book[50]['orders']['t1_1']['quantity'],
1)
self.assertTrue(self.ex1._bid_book_prices)
# remove remainder
self.ex1._modify_order('buy', 1, 't1_1', 50)
self.assertFalse(self.ex1._bid_book_prices)
self.assertEqual(self.ex1._bid_book[50]['num_orders'], 0)
self.assertEqual(self.ex1._bid_book[50]['size'], 0)
self.assertFalse('t1_1' in self.ex1._bid_book[50]['orders'].keys())
# Sell order
q2 = {
'order_id': 't1_1',
'timestamp': 5,
'type': 'add',
'quantity': 2,
'side': 'sell',
'price': 50
}
self.ex1.add_order_to_book(q2)
self.assertEqual(self.ex1._ask_book[50]['size'], 2)
# remove 1
self.ex1._modify_order('sell', 1, 't1_1', 50)
self.assertEqual(self.ex1._ask_book[50]['size'], 1)
self.assertEqual(self.ex1._ask_book[50]['orders']['t1_1']['quantity'],
1)
self.assertTrue(self.ex1._ask_book_prices)
# remove remainder
self.ex1._modify_order('sell', 1, 't1_1', 50)
self.assertFalse(self.ex1._ask_book_prices)
self.assertEqual(self.ex1._ask_book[50]['num_orders'], 0)
self.assertEqual(self.ex1._ask_book[50]['size'], 0)
self.assertFalse('t1_1' in self.ex1._ask_book[50]['orders'].keys())
def test_add_trade_to_book(self):
'''
add_trade_to_book() impacts trade_book
Check trade book empty, add a trade, check non-empty, verify dict equality
'''
t1 = dict(resting_order_id='t1_1',
resting_timestamp=2,
incoming_order_id='t2_1',
timestamp=5,
price=50,
quantity=1,
side='buy')
self.assertFalse(self.ex1.trade_book)
self.ex1._add_trade_to_book('t1_1', 2, 't2_1', 5, 50, 1, 'buy')
self.assertTrue(self.ex1.trade_book)
self.assertDictEqual(t1, self.ex1.trade_book[0])
def test_confirm_trade(self):
'''
confirm_trade() impacts confirm_trade_collector
Check confirm trade collector empty, add a trade, check non-empty, verify dict equality
'''
t2 = dict(timestamp=5,
trader='t3',
order_id='t3_1',
quantity=1,
side='sell',
price=50)
self.assertFalse(self.ex1.confirm_trade_collector)
self.ex1._confirm_trade(5, 'sell', 1, 't3_1', 50)
self.assertTrue(self.ex1.confirm_trade_collector)
self.assertDictEqual(t2, self.ex1.confirm_trade_collector[0])
def test_confirm_modify(self):
'''
confirm_modify() impacts confirm_modify_collector
Check confirm modify collector empty, add a trade, check non-empty, verify dict equality
'''
m1 = dict(timestamp=7,
trader='t5',
order_id='t5_10',
quantity=5,
side='buy')
self.assertFalse(self.ex1.confirm_modify_collector)
self.ex1._confirm_modify(7, 'buy', 5, 't5_10')
self.assertTrue(self.ex1.confirm_modify_collector)
self.assertDictEqual(m1, self.ex1.confirm_modify_collector[0])
def test_process_order(self):
'''
process_order() impacts confirm_modify_collector, traded indicator, order_history,
_bid_book and _bid_book_prices or _ask_book and _ask_book_prices.
process_order() is a traffic manager. An order is either an add order or not. If it is an add order,
it is either priced to go directly to the book or is sent to match_trade (which is tested below). If it
is not an add order, it is either modified or cancelled. To test, we will add some buy and sell orders,
then test for trades, cancels and modifies. process_order() also resets some object collectors.
'''
self.q2_buy['quantity'] = 2
self.q2_sell['quantity'] = 2
self.assertEqual(len(self.ex1._ask_book_prices), 0)
self.assertEqual(len(self.ex1._bid_book_prices), 0)
self.assertFalse(self.ex1.confirm_modify_collector)
self.assertFalse(self.ex1.order_history)
self.assertFalse(self.ex1.traded)
# seed order book
self.ex1.add_order_to_book(self.q1_buy)
self.ex1.add_order_to_book(self.q1_sell)
# process new orders
self.ex1.process_order(self.q2_buy)
self.ex1.process_order(self.q2_sell)
self.assertEqual(len(self.ex1._ask_book_prices), 1)
self.assertEqual(len(self.ex1._bid_book_prices), 1)
self.assertEqual(len(self.ex1.order_history), 2)
# marketable sell takes out 1 share
q3_sell = {
'order_id': 't3_1',
'timestamp': 5,
'type': 'add',
'quantity': 1,
'side': 'sell',
'price': 0
}
self.ex1.process_order(q3_sell)
self.assertEqual(len(self.ex1.order_history), 3)
self.assertEqual(self.ex1._bid_book[50]['num_orders'], 1)
self.assertEqual(self.ex1._bid_book[50]['size'], 2)
self.assertTrue(self.ex1.traded)
# marketable buy takes out 1 share
q3_buy = {
'order_id': 't3_2',
'timestamp': 5,
'type': 'add',
'quantity': 1,
'side': 'buy',
'price': 10000
}
self.ex1.process_order(q3_buy)
self.assertEqual(len(self.ex1.order_history), 4)
self.assertEqual(self.ex1._ask_book[52]['num_orders'], 1)
self.assertEqual(self.ex1._ask_book[52]['size'], 2)
self.assertTrue(self.ex1.traded)
# add/cancel buy order
q4_buy = {
'order_id': 't4_1',
'timestamp': 10,
'type': 'add',
'quantity': 1,
'side': 'buy',
'price': 48
}
self.ex1.process_order(q4_buy)
self.assertEqual(len(self.ex1.order_history), 5)
self.assertEqual(len(self.ex1._bid_book_prices), 2)
self.assertEqual(self.ex1._bid_book[48]['num_orders'], 1)
self.assertEqual(self.ex1._bid_book[48]['size'], 1)
self.assertFalse(self.ex1.traded)
q4_cancel1 = {
'order_id': 't4_1',
'timestamp': 10,
'type': 'cancel',
'quantity': 1,
'side': 'buy',
'price': 48
}
self.ex1.process_order(q4_cancel1)
self.assertEqual(len(self.ex1.order_history), 6)
self.assertEqual(len(self.ex1._bid_book_prices), 1)
self.assertFalse(self.ex1.traded)
# add/cancel sell order
q4_sell = {
'order_id': 't4_2',
'timestamp': 10,
'type': 'add',
'quantity': 1,
'side': 'sell',
'price': 54
}
self.ex1.process_order(q4_sell)
self.assertEqual(len(self.ex1.order_history), 7)
self.assertEqual(len(self.ex1._ask_book_prices), 2)
self.assertEqual(self.ex1._ask_book[54]['num_orders'], 1)
self.assertEqual(self.ex1._ask_book[54]['size'], 1)
self.assertFalse(self.ex1.traded)
q4_cancel2 = {
'order_id': 't4_2',
'timestamp': 10,
'type': 'cancel',
'quantity': 1,
'side': 'sell',
'price': 54
}
self.ex1.process_order(q4_cancel2)
self.assertEqual(len(self.ex1.order_history), 8)
self.assertEqual(len(self.ex1._ask_book_prices), 1)
self.assertFalse(self.ex1.traded)
# add/modify buy order
q5_buy = {
'order_id': 't5_1',
'timestamp': 10,
'type': 'add',
'quantity': 5,
'side': 'buy',
'price': 48
}
self.ex1.process_order(q5_buy)
self.assertEqual(len(self.ex1.order_history), 9)
self.assertEqual(len(self.ex1._bid_book_prices), 2)
self.assertEqual(self.ex1._bid_book[48]['num_orders'], 1)
self.assertEqual(self.ex1._bid_book[48]['size'], 5)
q5_modify1 = {
'order_id': 't5_1',
'timestamp': 10,
'type': 'modify',
'quantity': 2,
'side': 'buy',
'price': 48
}
self.ex1.process_order(q5_modify1)
self.assertEqual(len(self.ex1.order_history), 10)
self.assertEqual(len(self.ex1._bid_book_prices), 2)
self.assertEqual(self.ex1._bid_book[48]['size'], 3)
self.assertEqual(self.ex1._bid_book[48]['orders']['t5_1']['quantity'],
3)
self.assertEqual(len(self.ex1.confirm_modify_collector), 1)
self.assertFalse(self.ex1.traded)
# add/modify sell order
q5_sell = {
'order_id': 't5_1',
'timestamp': 10,
'type': 'add',
'quantity': 5,
'side': 'sell',
'price': 54
}
self.ex1.process_order(q5_sell)
self.assertEqual(len(self.ex1.order_history), 11)
self.assertEqual(len(self.ex1._ask_book_prices), 2)
self.assertEqual(self.ex1._ask_book[54]['num_orders'], 1)
self.assertEqual(self.ex1._ask_book[54]['size'], 5)
q5_modify2 = {
'order_id': 't5_1',
'timestamp': 10,
'type': 'modify',
'quantity': 2,
'side': 'sell',
'price': 54
}
self.ex1.process_order(q5_modify2)
self.assertEqual(len(self.ex1.order_history), 12)
self.assertEqual(len(self.ex1._ask_book_prices), 2)
self.assertEqual(self.ex1._ask_book[54]['size'], 3)
self.assertEqual(self.ex1._ask_book[54]['orders']['t5_1']['quantity'],
3)
self.assertEqual(len(self.ex1.confirm_modify_collector), 1)
self.assertFalse(self.ex1.traded)
def test_match_trade_sell(self):
'''
An incoming order can:
1. take out part of an order,
2. take out an entire price level,
3. if priced, take out a price level and make a new inside market.
'''
# seed order book
self.ex1.add_order_to_book(self.q1_buy)
self.ex1.add_order_to_book(self.q1_sell)
# process new orders
self.ex1.process_order(self.q2_buy)
self.ex1.process_order(self.q2_sell)
self.ex1.process_order(self.q3_buy)
self.ex1.process_order(self.q3_sell)
self.ex1.process_order(self.q4_buy)
self.ex1.process_order(self.q4_sell)
# The book: bids: 2@50, 3@49, 3@47 ; asks: 2@52, 3@53, 3@55
self.assertEqual(self.ex1._bid_book[47]['size'], 3)
self.assertEqual(self.ex1._bid_book[49]['size'], 3)
self.assertEqual(self.ex1._bid_book[50]['size'], 2)
self.assertEqual(self.ex1._ask_book[52]['size'], 2)
self.assertEqual(self.ex1._ask_book[53]['size'], 3)
self.assertEqual(self.ex1._ask_book[55]['size'], 3)
#self.assertFalse(self.ex1.sip_collector)
# market sell order takes out part of first best bid
q1 = {
'order_id': 't100_1',
'timestamp': 10,
'type': 'add',
'quantity': 1,
'side': 'sell',
'price': 0
}
self.ex1.process_order(q1)
self.assertEqual(self.ex1._bid_book[50]['size'], 1)
self.assertTrue(50 in self.ex1._bid_book_prices)
self.assertEqual(self.ex1._bid_book[49]['size'], 3)
self.assertEqual(self.ex1._bid_book[47]['size'], 3)
self.assertEqual(
self.ex1._bid_book[50]['orders'][self.ex1._bid_book[50]
['order_ids'][0]]['quantity'], 1)
#self.assertEqual(len(self.ex1.sip_collector), 1)
# market sell order takes out remainder first best bid and all of the next level
self.assertEqual(len(self.ex1._bid_book_prices), 3)
q2 = {
'order_id': 't100_2',
'timestamp': 11,
'type': 'add',
'quantity': 4,
'side': 'sell',
'price': 0
}
self.ex1.process_order(q2)
self.assertEqual(len(self.ex1._bid_book_prices), 1)
self.assertFalse(50 in self.ex1._bid_book_prices)
self.assertFalse(49 in self.ex1._bid_book_prices)
self.assertTrue(47 in self.ex1._bid_book_prices)
#self.assertEqual(len(self.ex1.sip_collector), 3)
# make new market
q3 = {
'order_id': 't101_1',
'timestamp': 12,
'type': 'add',
'quantity': 2,
'side': 'buy',
'price': 48
}
q4 = {
'order_id': 't102_1',
'timestamp': 13,
'type': 'add',
'quantity': 3,
'side': 'sell',
'price': 48
}
self.ex1.process_order(q3)
self.assertEqual(len(self.ex1._bid_book_prices), 2)
self.assertTrue(48 in self.ex1._bid_book_prices)
self.assertTrue(47 in self.ex1._bid_book_prices)
self.assertEqual(self.ex1._bid_book_prices[-1], 48)
self.assertEqual(self.ex1._bid_book_prices[-2], 47)
# sip_collector does not reset until new trade at new time
#self.assertEqual(len(self.ex1.sip_collector), 3)
self.ex1.process_order(q4)
self.assertEqual(len(self.ex1._bid_book_prices), 1)
self.assertFalse(48 in self.ex1._bid_book_prices)
self.assertTrue(47 in self.ex1._bid_book_prices)
self.assertEqual(len(self.ex1._ask_book_prices), 4)
self.assertTrue(48 in self.ex1._ask_book_prices)
self.assertEqual(self.ex1._ask_book_prices[0], 48)
self.assertEqual(self.ex1._bid_book_prices[-1], 47)
#self.assertEqual(len(self.ex1.sip_collector), 1)
def test_match_trade_buy(self):
'''
An incoming order can:
1. take out part of an order,
2. take out an entire price level,
3. if priced, take out a price level and make a new inside market.
'''
# seed order book
self.ex1.add_order_to_book(self.q1_buy)
self.ex1.add_order_to_book(self.q1_sell)
# process new orders
self.ex1.process_order(self.q2_buy)
self.ex1.process_order(self.q2_sell)
self.ex1.process_order(self.q3_buy)
self.ex1.process_order(self.q3_sell)
self.ex1.process_order(self.q4_buy)
self.ex1.process_order(self.q4_sell)
# The book: bids: 2@50, 3@49, 3@47 ; asks: 2@52, 3@53, 3@55
self.assertEqual(self.ex1._bid_book[47]['size'], 3)
self.assertEqual(self.ex1._bid_book[49]['size'], 3)
self.assertEqual(self.ex1._bid_book[50]['size'], 2)
self.assertEqual(self.ex1._ask_book[52]['size'], 2)
self.assertEqual(self.ex1._ask_book[53]['size'], 3)
self.assertEqual(self.ex1._ask_book[55]['size'], 3)
# market buy order takes out part of first best ask
q1 = {
'order_id': 't100_1',
'timestamp': 10,
'type': 'add',
'quantity': 1,
'side': 'buy',
'price': 100000
}
self.ex1.process_order(q1)
self.assertEqual(self.ex1._ask_book[52]['size'], 1)
self.assertTrue(52 in self.ex1._ask_book_prices)
self.assertEqual(self.ex1._ask_book[53]['size'], 3)
self.assertEqual(self.ex1._ask_book[55]['size'], 3)
self.assertEqual(
self.ex1._ask_book[52]['orders'][self.ex1._ask_book[52]
['order_ids'][0]]['quantity'], 1)
# market buy order takes out remainder first best ask and all of the next level
self.assertEqual(len(self.ex1._ask_book_prices), 3)
q2 = {
'order_id': 't100_2',
'timestamp': 11,
'type': 'add',
'quantity': 4,
'side': 'buy',
'price': 100000
}
self.ex1.process_order(q2)
self.assertEqual(len(self.ex1._ask_book_prices), 1)
self.assertFalse(52 in self.ex1._ask_book_prices)
self.assertFalse(53 in self.ex1._ask_book_prices)
self.assertTrue(55 in self.ex1._ask_book_prices)
# make new market
q3 = {
'order_id': 't101_1',
'timestamp': 12,
'type': 'add',
'quantity': 2,
'side': 'sell',
'price': 54
}
q4 = {
'order_id': 't102_1',
'timestamp': 13,
'type': 'add',
'quantity': 3,
'side': 'buy',
'price': 54
}
self.ex1.process_order(q3)
self.assertEqual(len(self.ex1._ask_book_prices), 2)
self.assertTrue(55 in self.ex1._ask_book_prices)
self.assertTrue(54 in self.ex1._ask_book_prices)
self.assertEqual(self.ex1._ask_book_prices[0], 54)
self.assertEqual(self.ex1._ask_book_prices[1], 55)
self.ex1.process_order(q4)
self.assertEqual(len(self.ex1._ask_book_prices), 1)
self.assertFalse(54 in self.ex1._ask_book_prices)
self.assertTrue(55 in self.ex1._ask_book_prices)
self.assertEqual(len(self.ex1._bid_book_prices), 4)
self.assertTrue(54 in self.ex1._bid_book_prices)
self.assertEqual(self.ex1._ask_book_prices[0], 55)
self.assertEqual(self.ex1._bid_book_prices[-1], 54)
def test_report_top_of_book(self):
'''
At setup(), top of book has 2 to sell at 52 and 2 to buy at 50
at time = 3
'''
self.ex1.add_order_to_book(self.q1_buy)
self.ex1.add_order_to_book(self.q2_buy)
self.ex1.add_order_to_book(self.q1_sell)
self.ex1.add_order_to_book(self.q2_sell)
tob_check = {
'timestamp': 5,
'best_bid': 50,
'best_ask': 52,
'bid_size': 2,
'ask_size': 2
}
self.ex1.report_top_of_book(5)
self.assertDictEqual(self.ex1._sip_collector[0], tob_check)
def test_market_collapse(self):
'''
At setup(), there is 8 total bid size and 8 total ask size
A trade for 8 or more should collapse the market
'''
print('Market Collapse Tests to stdout:\n')
# seed order book
self.ex1.add_order_to_book(self.q1_buy)
self.ex1.add_order_to_book(self.q1_sell)
# process new orders
self.ex1.process_order(self.q2_buy)
self.ex1.process_order(self.q2_sell)
self.ex1.process_order(self.q3_buy)
self.ex1.process_order(self.q3_sell)
self.ex1.process_order(self.q4_buy)
self.ex1.process_order(self.q4_sell)
# The book: bids: 2@50, 3@49, 3@47 ; asks: 2@52, 3@53, 3@55
# market buy order takes out part of the asks: no collapse
q1 = {
'order_id': 't100_1',
'timestamp': 10,
'type': 'add',
'quantity': 4,
'side': 'buy',
'price': 100000
}
self.ex1.process_order(q1)
# next market buy order takes out the asks: market collapse
q2 = {
'order_id': 't100_2',
'timestamp': 10,
'type': 'add',
'quantity': 5,
'side': 'buy',
'price': 100000
}
self.ex1.process_order(q2)
# market sell order takes out part of the bids: no collapse
q3 = {
'order_id': 't100_3',
'timestamp': 10,
'type': 'add',
'quantity': 4,
'side': 'sell',
'price': 0
}
self.ex1.process_order(q3)
# next market sell order takes out the asks: market collapse
q4 = {
'order_id': 't100_4',
'timestamp': 10,
'type': 'add',
'quantity': 5,
'side': 'sell',
'price': 0
}
self.ex1.process_order(q4)
if len(self.ob.book[symbol]['B']) > 0 and len(
self.ob.book[symbol]['S']) > 0:
self.strat.store_price(order, self.ob.book)
print(
'best bids/offers',
self.strat.previous_orders_highest[
order['Symbol']],
self.strat.previous_orders_lowest[order['Symbol']])
outcome = self.om.trade_after_news(order, self.strat)
self.send(outcome, order)
self.ob.clear_crossed_book(order, self.strat)
#print(self.ob.book)
else:
print('no trade, just modify')
self.strat.store_signal(order)
print(self.strat.signal_table[order['Symbol']])
finally:
print('The End!')
if __name__ == '__main__':
the_strat = Strategy()
the_om = OrderManager()
the_ob = Orderbook()
the_client = ClientConnection(the_ob, the_om, the_strat)
the_client.start()
print('Loaded model "' + name + '" from disk')
return model
def saveModel(model, name):
# serialize model to JSON
model_json = model.to_json()
with open(name + '.json', "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights(name + '.h5')
print('Saved model "' + name + '" to disk')
# Load orderbook
orderbook = Orderbook()
orderbook.loadFromEvents('ob-1.tsv')
orderbook_test = orderbook
orderbook.summary()
# import datetime
# orderbook = Orderbook()
# config = {
# 'startPrice': 10000.0,
# 'endPrice': 9940.0,
# 'levels': 25,
# 'qtyPosition': 0.1,
# 'startTime': datetime.datetime.now(),
# 'duration': datetime.timedelta(minutes=30),
# 'interval': datetime.timedelta(seconds=1)
# }
def consumer(self, marketData):
connection = elasticsearch.Elasticsearch(self.esHost)
self.ensure(connection)
self.docMappings(connection)
dataSet = json.loads(marketData)
item = {}
for infoPoint in dataSet:
try:
channel = str(infoPoint["channel"])
regex = "ok_sub_(spotusd|futureusd)_(b|l)tc_(.[A-Za-z0-9_]+)"
search = re.search(regex, channel)
if search.group(1) == "futureusd":
isFuture = True
else:
isFuture = False
currencyPair = str(search.group(2)) + "tc_usd"
self.count = self.count + 1
if self.count % 100 == 0:
print ("PROCESSED " + str(self.count) + " DATA POINTS SO FAR...")
if search.group(3) == "index":
myindex = FutureIndex()
dto = myindex.getFutureIndexDto(infoPoint, currencyPair)
dto["exchange"] = "OKCOIN"
self.postDto(dto, connection, "future_price_index")
elif "depth" in channel:
mybook = Orderbook()
dto = mybook.getDepthDtoList(infoPoint, currencyPair, isFuture)
for item in dto:
item["websocket_name"] = channel
item["is_future"] = isFuture
if isFuture == True:
check = re.search("depth_(this_week|next_week|quarter)_(20|60)", search.group(3).strip())
item["contract_type"] = str(check.group(1))
item["depth"] = str(check.group(2))
else:
item["contract_type"] = "spot"
depthSearch = re.search("depth_(20|60)", search.group(3).strip())
item["depth"] = depthSearch.group(1)
item["exchange"] = "OKCOIN"
self.postDto(item, connection, "orderbook")
elif "ticker" in channel and "data" in infoPoint:
myticker = Ticker()
if isFuture == False:
dto = myticker.getTickerDto(infoPoint, currencyPair)
self.postDto(dto, connection, "ticker")
elif isFuture == True:
dto = myticker.getFutureTickerDto(infoPoint, channel, currencyPair)
dto["exchange"] = "OKCOIN"
self.postDto(dto, connection, "future_ticker")
elif "trade" in channel:
mytrade = Trade()
if "data" in infoPoint:
dtoList = mytrade.getCompletedTradeDtoList(infoPoint, currencyPair)
for item in dtoList:
item["is_future"] = "futureusd" in channel
item["websocket_name"] = channel
item["exchange"] = "OKCOIN"
self.postDto(item, connection, "completed_trades")
elif "kline" in channel:
myklein = KlineCandle()
if "data" in infoPoint:
if len(infoPoint["data"]) > 1:
for klineData in infoPoint["data"]:
if type(klineData) is list:
klineDto = myklein.getKlineDto(klineData, currencyPair, channel)
klineDto["exchange"] = "OKCOIN"
klineDto["is_future"] = isFuture
klineDto["websocket_name"] = channel
else:
klineDto = myklein.getKlineDto(infoPoint["data"], currencyPair, channel)
self.postDto(klineDto, connection, "kline_candles")
except:
raise
from order_side import OrderSide
from orderbook import Orderbook
orderbook = Orderbook()
orderbook.loadFromEvents('ob-1-small.tsv')
orderbook_test = orderbook
orderbook.summary()
side = OrderSide.SELL
levels = list(range(-20, 21))
episode = {
'episode': 9,
'steps': {
0: {
'action': 2,
'index': 167,
't': 100,
'i': 0.9999999999999999,
'reward': -6.224232140000822
},
1: {
'action': 39,
'index': 173,
't': 90,
'i': 0.9999999999999999,
'reward': 1.9899999999997817
},
2: {
'action': 21,
'index': 179,
"Price": "113.36"
}
test_2 = {
"Symbol": "ADBE",
"OrderID": "002",
"Action": "A",
"Exchange": "1",
"Quantity": "329000",
"News": "50",
"Side": "S",
"Description": "Adobe",
"Price": "118.41"
}
order_book_1 = Orderbook()
order_book_1.add(test_1)
# print(Orderbook.all)
order_book_1.add(test_2)
#print(Orderbook.all)
assert len(order_book_1.book['ADBE']['B']) == 1
assert len(order_book_1.book['ADBE']['S']) == 1
test_3 = {
"Symbol": "ADBE",
"OrderID": "002",
"Action": "M",
"Exchange": "1",
"Quantity": "362000",
"News": "0",
levels = [5, 4, 3, 2, 1, 0, -1, -2, -3, -4, -5, -6, -7, -10, -12, -15]
ai = QLearn(actions=levels, epsilon=0.4, alpha=0.3, gamma=0.8)
#trainBook = 'query_result_train_15m.tsv'
#testBook = 'query_result_train_15m.tsv'
# orderbook = Orderbook(extraFeatures=False)
# orderbook.loadFromBitfinexFile('orderbook_bitfinex_btcusd_view.tsv')
# orderbook_test = Orderbook(extraFeatures=False)
# orderbook_test.loadFromBitfinexFile('orderbook_bitfinex_btcusd_view.tsv')
# Load orderbook
cols = ["ts", "seq", "size", "price", "is_bid", "is_trade", "ttype"]
import pandas as pd
events = pd.read_table('ob-1-small.tsv', sep='\t', names=cols, index_col="seq")
d = Orderbook.generateDictFromEvents(events)
orderbook = Orderbook()
orderbook.loadFromDict(d)
# clean first n states (due to lack of bids and asks)
print("#States: " + str(len(orderbook.states)))
for i in range(100):
orderbook.states.pop(0)
del d[list(d.keys())[0]]
orderbook_test = orderbook
#orderbook.plot()
T = [0, 10, 20, 40, 60, 80, 100] #, 120, 240]
T_test = [0, 10, 20, 40, 60, 80, 100] # 120, 240]
I = [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
actionSpace = ActionSpace(orderbook, side, T, I, ai, levels)
def __init__(self, summary, set_description):
self.item_id = set_description['id']
self.etf_name = set_description['name']
self.orderbook = Orderbook(summary, self.item_id)
self.parts = [Orderbook(summary, part['id']) for part in set_description['parts']]
def compute_spread(market):
dcx_ob = dcx.get_orderbook(market=market)
ob = Orderbook()
ob.init(dcx_orderbook=dcx_ob)
print(f'spread for market: {market} is {ob.spread}')
from orderbook import Orderbook
import numpy as np
import pandas as pd
import os
book = 'query_result_test.tsv'
tmp = 'feature.tsv'
orderbook = Orderbook()
orderbook.loadFromFile(book)
states = orderbook.getStates()
def stateDiff(start, end):
"""Calculate time difference between two states."""
consumed = (end.getTimestamp() - start.getTimestamp()).total_seconds()
return consumed
def getPastState(i, t):
"""Find state at index i - time t."""
endState = states[i]
state = endState
while (stateDiff(state, endState) < t):
i = i - 1
if i < 0:
raise Exception("Not enough states available for diff.")
state = states[i]
return i
def traverse(f, g, default=0.0, t=60):
