def test_skipped(self):
a1 = Asset('FX:EURUSD')
a2 = Asset('STK:AAPL')
bars = BarCoordinator([a1, a2], const.Freq.MINUTE5, const.Freq.SECOND5)
self.assertIsNone(
bars.update(
Datum(a1, pd.Timestamp('2019-01-01 12:00:05'), 2, 3, 0, 1)))
self.assertIsNone(
bars.update(
Datum(a2, pd.Timestamp('2019-01-01 12:00:05'), 1, 4, -1, 1)))
self.assertIsNone(
bars.update(
Datum(a1, pd.Timestamp('2019-01-01 12:00:10'), 1, 4, 1, 2)))
self.assertIsNone(
bars.update(
Datum(a2, pd.Timestamp('2019-01-01 12:05:00'), 1, 3, -2, 0)))
self.assertIn(a2, bars.ready)
out = bars.update(
Datum(a1, pd.Timestamp('2019-01-01 12:05:10'), 2, 3, 0, 1))
self.assertEqual(out[0], pd.Timestamp('2019-01-01 12:05:00'))
assert_array_equal(out[1], [2, 1]) # open
assert_array_equal(out[2], [4, 4]) # high
assert_array_equal(out[3], [0, -2]) # low
assert_array_equal(out[4], [2, 0]) # close
self.assertEqual(bars.check, pd.Timestamp('2019-01-01 12:10:00'))
self.assertEqual(bars.timestamp, pd.Timestamp('2019-01-01 12:10:00'))
self.assertEqual(bars.data[a1].open, 2)
self.assertEqual(bars.data[a2].open, None)
def test_offset(self):
asset = Asset('FX:EURUSD')
now = pd.Timestamp('2019-01-01 12:00:55') # start from the last bar
bar = DownSampledBar(const.Freq.MINUTE, const.Freq.SECOND5, offset=1)
# test start
out = bar.update(Datum(asset, now, 1, 3, 0, 2))
self.assertTupleEqual(
out, (pd.Timestamp('2019-01-01 12:01:00'), 1, 3, 0, 2))
# test update
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:01:00'), 1, 3, 0, 2)))
out = bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:01:55'), 6, 9, 4, 5))
self.assertTupleEqual(
out, (pd.Timestamp('2019-01-01 12:02:00'), 1, 9, 0, 5))
self.assertEqual(bar.check, pd.Timestamp.min)
self.assertIsNone(bar.open)
# test skip
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:03:00'), 6, 9, 4, 5)))
out = bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:04:00'), 1, 3, 0, 2))
self.assertEqual(out,
(pd.Timestamp('2019-01-01 12:04:00'), 6, 9, 4, 5))
self.assertEqual(bar.timestamp, pd.Timestamp('2019-01-01 12:05:00'))
self.assertEqual(bar.open, 1)
self.assertEqual(bar.close, 2)
def test_5minute(self):
asset = Asset('FX:EURUSD')
bar = DownSampledBar(const.Freq.MINUTE5, const.Freq.SECOND5, offset=1)
# test start
out = bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:04:55'), 1, 3, 0, 2))
self.assertTupleEqual(
out, (pd.Timestamp('2019-01-01 12:05:00'), 1, 3, 0, 2))
# test update
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:05:00'), 1, 3, 0, 2)))
out = bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:09:55'), 6, 9, 4, 5))
self.assertTupleEqual(
out, (pd.Timestamp('2019-01-01 12:10:00'), 1, 9, 0, 5))
self.assertEqual(bar.check, pd.Timestamp.min)
self.assertIsNone(bar.open)
# test skip
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:10:00'), 6, 9, 4, 5)))
out = bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:15:00'), 1, 3, 0, 2))
self.assertEqual(out,
(pd.Timestamp('2019-01-01 12:15:00'), 6, 9, 4, 5))
self.assertEqual(bar.timestamp, pd.Timestamp('2019-01-01 12:20:00'))
self.assertEqual(bar.open, 1)
self.assertEqual(bar.close, 2)
def test_start(self):
now = pd.Timestamp.now()
print(now)
data = Datum(Asset('FX:EURUSD'), now, 1, 2, 3, 4)
bar = DownSampledBar(const.Freq.MINUTE, const.Freq.SECOND5)
out = bar.update(data)
self.assertIsNone(out)
self.assertEqual(bar.timestamp, now.ceil('1min'),
'Internal timestamp should be the ceiling value')
self.assertEqual(bar.check, now.ceil('1min'),
'"Check" timestamp should be the ceiling value')
self.assertEqual(bar.open, 1)
self.assertEqual(bar.close, 4)
bar = DownSampledBar(const.Freq.MINUTE, const.Freq.SECOND5, 2)
bar.update(data)
if now.second > 50:
self.assertEqual(bar.timestamp,
(now + pd.Timedelta(seconds=10)).ceil('1min'),
'Offset test wrapped')
self.assertEqual(bar.check,
(now + pd.Timedelta(seconds=10)).ceil('1min') -
pd.Timedelta(seconds=10))
else:
self.assertEqual(bar.timestamp, now.ceil('1min'), 'Offset test')
self.assertEqual(bar.check,
now.ceil('1min') - pd.Timedelta(seconds=10))
self.assertEqual(bar.high, 2)
def test_insertion(self):
asset = Asset('FX:ABC')
store = DataStore(
Broker.MOCK,
Freq.MINUTE) # tick is not used and for testing purpose
now = pd.Timestamp.utcnow()
async def main():
await store.connect()
for i in range(5):
data = Datum(asset, now + pd.Timedelta(minutes=i), 1, 2, 3, 4)
await store.append(data)
await store.disconnect()
asyncio.run(main())
database = Config['postgres_db']
user = Config['postgres_user']
password = Config['postgres_pass']
port = Config['postgres_port']
con = pg.connect(database=database,
user=user,
password=password,
host='127.0.0.1',
port=port)
cur = con.cursor()
cur.execute(f'select * from {store.table_name}')
res = cur.fetchall()
self.assertEqual(res[0][1].astimezone(timezone('EST')),
now.astimezone(timezone('EST')))
self.assertIsInstance(res[0][2], datetime)
self.assertTupleEqual(res[0][3:], (1.0, 2.0, 3.0, 4.0))
cur.execute(f'drop table {store.table_name}')
con.commit()
con.close()
async def process_subscription(self, socket: azmq.Socket):
"""
Receive request of the form "asset1:ticker1,asset2:ticker2,..."
Subscribe to the broker
TODO: add handling for exception. Currently always succeed
"""
msg = await socket.recv_string(
) # format: asset1:ticker1,asset2:ticker2,...
self.logger.info(f'Receive subscription request on {msg}')
# filter existing assets
asset_strings = msg.split(',')
assets = [Asset(s) for s in asset_strings]
assets = [asset for asset in assets if asset not in self.subscribed]
if len(assets) > 0:
# submit subscription
await utils.wait_for(self.subscribe(assets),
self.subscription_wait_time,
JobError('Data subscription timeout'))
# remember newly added subscription
self.subscribed.update(dict.fromkeys(assets))
self.logger.info(f'Subscribed: {str(assets)}')
else:
self.logger.info(f'No new subscription is needed')
socket.send_json({'code': const.DCode.Succeeded.value})
def test_offset(self):
a1 = Asset('FX:EURUSD')
a2 = Asset('STK:AAPL')
bars = BarCoordinator([a1, a2], const.Freq.MINUTE5, const.Freq.SECOND5,
1)
self.assertIsNone(
bars.update(
Datum(a1, pd.Timestamp('2019-01-01 12:00:00'), 2, 3, 0, 1)))
self.assertIsNone(
bars.update(
Datum(a2, pd.Timestamp('2019-01-01 12:00:00'), 1, 4, -1, 1)))
self.assertIsNone(
bars.update(
Datum(a1, pd.Timestamp('2019-01-01 12:00:05'), 1, 4, 1, 2)))
self.assertIsNone(
bars.update(
Datum(a2, pd.Timestamp('2019-01-01 12:04:55'), 1, 3, -2, 0)))
self.assertIn(a2, bars.ready)
out = bars.update(
Datum(a1, pd.Timestamp('2019-01-01 12:04:55'), 2, 3, 0, 1))
self.assertEqual(out[0], pd.Timestamp('2019-01-01 12:05:00'))
assert_array_equal(out[1], [2, 1]) # open
assert_array_equal(out[2], [4, 4]) # high
assert_array_equal(out[3], [0, -2]) # low
assert_array_equal(out[4], [1, 0]) # close
self.assertIsNone(bars.check)
assert_array_equal(bars.opens, [np.nan, np.nan])
# test consecutive bars
self.assertIsNone(
bars.update(
Datum(a1, pd.Timestamp('2019-01-01 12:09:55'), 2, 3, 0, 1)))
out = bars.update(
Datum(a2, pd.Timestamp('2019-01-01 12:09:55'), 3, 4, 0, 2))
self.assertEqual(out[0], pd.Timestamp('2019-01-01 12:10:00'))
assert_array_equal(out[1], [2, 3]) # open
assert_array_equal(out[2], [3, 4]) # high
assert_array_equal(out[3], [0, 0]) # low
assert_array_equal(out[4], [1, 2]) # close
def test_equity(self):
a1 = 'STK:AAPL'
a2 = 'STK:MSFT'
assets = [Asset(a1), Asset(a2)]
equity = InMemoryEquityCurve(assets, 10000)
equity.update(pd.Timestamp('2019-01-01'), {
a1: [10, 200, 10],
a2: [100, 50, 5]
}, np.array([200, 50]))
equity.update(pd.Timestamp('2019-01-02'), {
a1: [-5, 250, 5],
a2: [-80, 45, 5]
}, np.array([250, 45]))
equity.update(pd.Timestamp('2019-01-02'), {a1: [-10, 220, 10]},
np.array([220, 60]))
assert_array_equal(equity.market_values,
[[0, 0, 10000], [2000, 5000, 3000],
[1250, 900, 7850], [-1100, 1200, 10050]])
assert_array_equal(equity.shares,
[[0, 0], [10, 100], [5, 20], [-5, 20]])
assert_array_equal(equity.commissions,
[[0, 0], [10, 5], [5, 5], [10, 0]])
def test_update(self):
asset = Asset('FX:EURUSD')
now = pd.Timestamp('2019-01-01 12:01:00')
data = Datum(asset, now, 1, 2, 3, 4)
bar = DownSampledBar(const.Freq.MINUTE, const.Freq.SECOND5)
# test immediate return
out = bar.update(data)
self.assertIsNotNone(out)
self.assertTupleEqual(out, (now, 1, 2, 3, 4))
# test regular update
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:01:05'), 1, 3, 0, 2)))
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:01:55'), 1, 6, 0, 2)))
out = bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:02:00'), 2, 5, 1, 1))
self.assertEqual(out,
(pd.Timestamp('2019-01-01 12:02:00'), 1, 6, 0, 1))
self.assertEqual(bar.check, pd.Timestamp.min)
self.assertEqual(bar.open, None)
# test consecutive update
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:02:05'), 1, 3, 0, 2)))
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:02:10'), 4, 5, -1,
3)))
out = bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:03:00'), 2, 4, 1, 1))
self.assertEqual(out,
(pd.Timestamp('2019-01-01 12:03:00'), 1, 5, -1, 1))
# test skip update
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:04:05'), 1, 3, 0, 2)))
out = bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:05:05'), 2, 4, 1,
1)) # skip from the 4th minute
self.assertTupleEqual(
out, (pd.Timestamp('2019-01-01 12:05:00'), 1, 3, 0, 2))
self.assertEqual(bar.timestamp, pd.Timestamp('2019-01-01 12:06:00'))
self.assertEqual(bar.open, 2)
self.assertEqual(bar.high, 4)
self.assertEqual(bar.close, 1)
def test_update(self):
a1 = Asset('FX:EURUSD')
a2 = Asset('STK:AAPL')
panel = BarPanel([a1, a2], const.Freq.MINUTE5, const.Freq.SECOND5, 2)
self.assertIsNone(
panel.update(
Datum(a1, pd.Timestamp('2019-01-01 12:05:00'), 1, 2, -1, 0)))
out = panel.update(
Datum(a2, pd.Timestamp('2019-01-01 12:05:00'), 1, 2, 0, 1))
self.assertEqual(out[0], pd.Timestamp('2019-01-01 12:05:00'))
assert_array_equal(out[1]['open'], [[1, 1]])
assert_array_equal(out[1]['high'], [[2, 2]])
assert_array_equal(out[1]['low'], [[-1, 0]])
assert_array_equal(out[1]['close'], [[0, 1]])
self.assertIsNone(
panel.update(
Datum(a1, pd.Timestamp('2019-01-01 12:10:00'), 2, 3, 0, 1)))
out = panel.update(
Datum(a2, pd.Timestamp('2019-01-01 12:10:00'), 1, 2, 0, 1))
self.assertEqual(out[0], pd.Timestamp('2019-01-01 12:10:00'))
assert_array_equal(out[1]['open'], [[1, 1], [2, 1]])
assert_array_equal(out[1]['high'], [[2, 2], [3, 2]])
assert_array_equal(out[1]['low'], [[-1, 0], [0, 0]])
assert_array_equal(out[1]['close'], [[0, 1], [1, 1]])
self.assertIsNone(
panel.update(
Datum(a1, pd.Timestamp('2019-01-01 12:15:00'), 2, 3, 0, 1)))
out = panel.update(
Datum(a2, pd.Timestamp('2019-01-01 12:15:00'), 1, 2, 0, 1))
self.assertEqual(out[0], pd.Timestamp('2019-01-01 12:15:00'))
assert_array_equal(out[1]['open'], [[2, 1], [2, 1]])
assert_array_equal(out[1]['high'], [[3, 2], [3, 2]])
assert_array_equal(out[1]['low'], [[0, 0], [0, 0]])
assert_array_equal(out[1]['close'], [[1, 1], [1, 1]])
async def execute_order(self, socket: azmq.Socket):
"""
TODO: spawn out task for each request so that we can attend to other requests while waiting for fills
We need an extra port to do this. Executor also need to provide a port
For, we handle order sequentially
"""
struct = await socket.recv_json() # type: dict
self.logger.info(f'Receive order: {struct}')
order = {Asset(k): v for k, v in struct.items() if k != 'Strategy'}
executions = await self.execute(struct['Strategy'], order)
self.logger.info(
f'Executed order for {struct["Strategy"]} {str(executions)}')
await socket.send_json(executions)
def test_high_water_mark(self):
asset = Asset('FX:EURUSD')
bar = DownSampledBar(const.Freq.MINUTE, const.Freq.SECOND5)
# test corner case where the starting point is the last bar
out = bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:01:00'), 1, 2, 3, 4))
self.assertTupleEqual(
out, (pd.Timestamp('2019-01-01 12:01:00'), 1, 2, 3, 4))
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:01:00'), 1, 2, 3, 4)))
self.assertEqual(bar.high_water_mark,
pd.Timestamp('2019-01-01 12:01:00'))
# test consecutive update
bar = DownSampledBar(const.Freq.MINUTE, const.Freq.SECOND5)
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:01:50'), 1, 2, 3, 4)))
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:01:55'), 1, 2, 3, 4)))
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:01:55'), 1, 10, 0,
2)))
self.assertEqual(bar.high, 2)
self.assertEqual(bar.low, 3)
self.assertEqual(bar.close, 4)
self.assertIsNotNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:02:00'), 1, 2, 3, 4)))
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:02:00'), 1, 2, 3, 4)))
self.assertIsNone(
bar.update(
Datum(asset, pd.Timestamp('2019-01-01 12:02:05'), 1, 2, 3, 4)))
# @schedulable()
# async def kill(self):
# await asyncio.sleep(duration)
# raise JobError('Killed')
def run_controller():
controller = Controller()
controller.run()
def run_data_server():
# noinspection PyArgumentList
server = IBDataServer()
server.run()
def run_executor():
time.sleep(1) # wait for server to start
executor = Executor(strategy, broker, 10000)
executor.run()
threads = [
threading.Thread(target=func)
for func in [run_data_server, run_controller, run_executor]
]
[t.start() for t in threads]
[t.join() for t in threads]
if __name__ == '__main__':
strat = RandomStrategy([Asset('FX:EURUSD')], const.Freq.MINUTE5, 10000)
run_servers(IBDataServer, const.Broker.IB, strat, 600)