async def connect(self):
with open(self._filename) as csvfile:
self._reader = csv.DictReader(csvfile, delimiter=',')
for row in self._reader:
order = Order(volume=float(row['volume']),
price=float(row['close']),
side=Side.BUY,
exchange=self.exchange(),
instrument=Instrument(
row['symbol'].split('-')[0],
InstrumentType(row['symbol'].split('-')[1].upper())
)
)
order.filled = float(row['volume'])
if 'time' in row:
order.timestamp = datetime.fromtimestamp(float(row['time']))
elif 'date' in row:
order.timestamp = datetime.fromisoformat(row['date'])
elif 'datetime' in row:
order.timestamp = datetime.fromisoformat(row['datetime'])
self._data.append(Trade(volume=float(row['volume']),
price=float(row['close']),
maker_orders=[],
taker_order=order))
async def tick(self):
now = self._start
for i in range(1000):
if self._client_order:
self._client_order.filled = self._client_order.volume
t = Trade(self._client_order.volume, i, [], self._client_order)
t.taker_order.timestamp = now
self._client_order = None
yield Event(type=EventType.TRADE, target=t)
continue
o = Order(1, i, Side.BUY, self._instrument, self.exchange())
o.filled = 1
o.timestamp = now
t = Trade(1, i, [], o)
yield Event(type=EventType.TRADE, target=t)
now += timedelta(minutes=30)
def add(self, order: Order) -> None:
"""add a new order to the order book, potentially triggering events:
EventType.TRADE: if this order crosses the book and fills orders
EventType.FILL: if this order crosses the book and fills orders
EventType.CHANGE: if this order crosses the book and partially fills orders
Args:
order (Data): order to submit to orderbook
"""
if order is None:
raise Exception("Order cannot be None")
# secondary triggered orders
secondaries: List[Order] = []
# get the top price on the opposite side of book
top = self._getTop(order.side, self._collector.clearedLevels())
# set levels to the right side
levels = self._buy_levels if order.side == Side.BUY else self._sell_levels
prices = self._buys if order.side == Side.BUY else self._sells
prices_cross = self._sells if order.side == Side.BUY else self._buys
# set order price appropriately
if order.order_type == OrderType.MARKET:
if order.flag in (None, OrderFlag.NONE):
# price goes infinite "fill however you want"
order_price = float(
"inf") if order.side == Side.BUY else float("-inf")
else:
# with a flag, the price dictates the "max allowed price" to AON or FOK under
order_price = order.price
else:
order_price = order.price
# check if crosses
while top is not None and (order_price >= top if order.side == Side.BUY
else order_price <= top):
# execute order against level
# if returns trade, it cleared the level
# else, order was fully executed
trade, new_secondaries = prices_cross[top].cross(order)
if new_secondaries:
# append to secondaries
secondaries.extend(new_secondaries)
if trade:
# clear sell level
top = self._getTop(
order.side, self._collector.clearLevel(prices_cross[top]))
continue
# trade is done, check if level was cleared exactly
if not prices_cross[top]:
# level cleared exactly
self._collector.clearLevel(prices_cross[top])
break
# if order remaining, check rules/push to book
if order.filled < order.volume:
if order.order_type == OrderType.MARKET:
# Market orders
if order.flag in (OrderFlag.ALL_OR_NONE,
OrderFlag.FILL_OR_KILL):
# cancel the order, do not execute any
self._collector.revert()
# cancel the order
self._collector.pushCancel(order)
self._collector.commit()
else:
# market order, partial
if order.filled > 0:
self._collector.pushTrade(order, order.filled)
# clear levels
self._clearOrders(order, self._collector.clearedLevels())
# execute order, cancel the rest
self._collector.pushCancel(order)
self._collector.commit()
# execute secondaries
for secondary in secondaries:
secondary.timestamp = order.timestamp # adjust trigger time
self.add(secondary)
else:
# Limit Orders
if order.flag == OrderFlag.FILL_OR_KILL:
if order.filled > 0:
# reverse partial
# cancel the order, do not execute any
self._collector.revert()
# reset filled
order.filled = 0.0
# cancel the order
self._collector.pushCancel(order)
self._collector.commit()
else:
# add to book
self._collector.commit()
# limit order, put on books
if _insort(levels, order.price):
# new price level
prices[order.price] = _PriceLevel( # type: ignore
order.price,
collector=self._collector)
# add order to price level
prices[order.price].add(order)
# execute secondaries
for secondary in secondaries:
secondary.timestamp = order.timestamp # adjust trigger time
self.add(secondary)
elif order.flag == OrderFlag.ALL_OR_NONE:
if order.filled > 0:
# order could not fill fully, revert
# cancel the order, do not execute any
self._collector.revert()
# reset filled
order.filled = 0.0
# cancel the order
self._collector.pushCancel(order)
self._collector.commit()
else:
# add to book
self._collector.commit()
# limit order, put on books
if _insort(levels, order.price):
# new price level
prices[order.price] = _PriceLevel( # type: ignore
order.price,
collector=self._collector)
# add order to price level
prices[order.price].add(order)
# execute secondaries
for secondary in secondaries:
secondary.timestamp = order.timestamp # adjust trigger time
self.add(secondary)
elif order.flag == OrderFlag.IMMEDIATE_OR_CANCEL:
if order.filled > 0:
# clear levels
self._clearOrders(order,
self._collector.clearedLevels())
# execute the ones that filled, kill the remainder
self._collector.pushCancel(order)
# commit
self._collector.commit()
# execute secondaries
for secondary in secondaries:
secondary.timestamp = order.timestamp # adjust trigger time
self.add(secondary)
else:
# add to book
self._collector.commit()
# limit order, put on books
if _insort(levels, order.price):
# new price level
prices[order.price] = _PriceLevel( # type: ignore
order.price,
collector=self._collector)
# add order to price level
prices[order.price].add(order)
# execute secondaries
for secondary in secondaries:
secondary.timestamp = order.timestamp # adjust trigger time
self.add(secondary)
else:
# clear levels
self._clearOrders(order, self._collector.clearedLevels())
# execute order
self._collector.commit()
# limit order, put on books
if _insort(levels, order.price):
# new price level
prices[order.price] = _PriceLevel( # type: ignore
order.price,
collector=self._collector)
# add order to price level
prices[order.price].add(order)
# execute secondaries
for secondary in secondaries:
secondary.timestamp = order.timestamp # adjust trigger time
self.add(secondary)
else:
if order.filled > order.volume:
raise Exception(
"Unknown error occurred - order book is corrupt")
# don't need to add trade as this is done in the price_levels
# clear levels
self._clearOrders(order, self._collector.clearedLevels())
# execute all the orders
self._collector.commit()
# execute secondaries
for secondary in secondaries:
secondary.timestamp = order.timestamp # adjust trigger time
self.add(secondary)
# clear the collector
self._collector.clear()
async def tick(self):
'''return data from exchange'''
if self._timeframe == 'live':
data = deque()
def _callback(record):
data.append(record)
self._client.tradesSSE(symbols=",".join(
[i.name for i in self._subscriptions]),
on_data=_callback)
while True:
while data:
record = data.popleft()
volume = record['volume']
price = record['price']
instrument = Instrument(record['symbol'],
InstrumentType.EQUITY)
o = Order(volume=volume,
price=price,
side=Side.BUY,
instrument=instrument,
exchange=self.exchange())
t = Trade(volume=volume,
price=price,
taker_order=o,
maker_orders=[])
yield Event(type=EventType.TRADE, target=t)
await asyncio.sleep(0)
else:
dfs = []
insts = set()
if self._timeframe != '1d':
for i in tqdm(self._subscriptions, desc="Fetching data..."):
if i.name in insts:
# already fetched the data, multiple subscriptions
continue
if self._cache_data:
# first, check if we have this data and its cached already
os.makedirs('_aat_data', exist_ok=True)
data_filename = os.path.join(
'_aat_data', 'iex_{}_{}_{}_{}.pkl'.format(
i.name, self._timeframe,
datetime.now().strftime('%Y%m%d'),
'sand' if self._is_sandbox else ''))
if os.path.exists(data_filename):
print('using cached IEX data for {}'.format(
i.name))
df = pd.read_pickle(data_filename)
else:
df = self._client.chartDF(
i.name, timeframe=self._timeframe)
df.to_pickle(data_filename)
else:
df = self._client.chartDF(i.name,
timeframe=self._timeframe)
df = df[['close', 'volume']]
df.columns = [
'close:{}'.format(i.name), 'volume:{}'.format(i.name)
]
dfs.append(df)
insts.add(i.name)
data = pd.concat(dfs, axis=1)
data.sort_index(inplace=True)
data = data.groupby(data.index).last()
data.drop_duplicates(inplace=True)
data.fillna(method='ffill', inplace=True)
else:
for i in tqdm(self._subscriptions, desc="Fetching data..."):
if i.name in insts:
# already fetched the data, multiple subscriptions
continue
date = self._start_date
subdfs = []
while date <= self._end_date:
if self._cache_data:
# first, check if we have this data and its cached already
os.makedirs('_aat_data', exist_ok=True)
data_filename = os.path.join(
'_aat_data', 'iex_{}_{}_{}_{}.pkl'.format(
i.name, self._timeframe, date,
'sand' if self._is_sandbox else ''))
if os.path.exists(data_filename):
print(
'using cached IEX data for {} - {}'.format(
i.name, date))
df = pd.read_pickle(data_filename)
else:
df = self._client.chartDF(
i.name,
timeframe='1d',
date=date.strftime('%Y%m%d'))
df.to_pickle(data_filename)
else:
df = self._client.chartDF(
i.name,
timeframe='1d',
date=date.strftime('%Y%m%d'))
if not df.empty:
df = df[['average', 'volume']]
df.columns = [
'close:{}'.format(i.name),
'volume:{}'.format(i.name)
]
subdfs.append(df)
date += timedelta(days=1)
dfs.append(pd.concat(subdfs))
insts.add(i.name)
data = pd.concat(dfs, axis=1)
data.index = [
x + timedelta(hours=int(y.split(':')[0]),
minutes=int(y.split(':')[1]))
for x, y in data.index
]
data = data.groupby(data.index).last()
data.drop_duplicates(inplace=True)
data.fillna(method='ffill', inplace=True)
for index in data.index:
for i in self._subscriptions:
volume = data.loc[index]['volume:{}'.format(i.name)]
price = data.loc[index]['close:{}'.format(i.name)]
if volume == 0:
continue
o = Order(volume=volume,
price=price,
side=Side.BUY,
instrument=i,
exchange=self.exchange())
o.filled = volume
o.timestamp = index.to_pydatetime()
t = Trade(volume=volume,
price=price,
taker_order=o,
maker_orders=[])
yield Event(type=EventType.TRADE, target=t)
await asyncio.sleep(0)
while self._queued_orders:
order = self._queued_orders.popleft()
order.timestamp = index
order.filled = order.volume
t = Trade(volume=order.volume,
price=order.price,
taker_order=order,
maker_orders=[])
t.my_order = order
yield Event(type=EventType.TRADE, target=t)
await asyncio.sleep(0)
