class cache:
def __init__(self, latest=False):
self.feedCache = Cache(".feedcache")
self.latest = latest
def __preprocess_title(self, feed):
for entry in feed.entries:
entry["feed_src"] = feed["feed"]["title"]
return feed
def __manage_cache(self, url):
try:
if url in self.feedCache:
data = self.feedCache.get(url)
else:
parsed_feed = feedparser.parse(url)
data = self.__preprocess_title(parsed_feed)
# cache expires in 30 mins
self.feedCache.add(url, data, expire=1800)
self.feedCache.close()
except ValueError:
pass
except Exception:
pass
return data
def get_feed(self, url):
if self.latest:
latestFeed = self.__manage_cache(url).entries
if len(latestFeed) > 0:
return latestFeed[0]
else:
pass
return self.__manage_cache(url).entries
class Orderbook(object):
def __init__(self, extraFeatures=False):
self.cache = Cache('/tmp/ctc-executioner')
self.dictBook = None
self.trades = {}
self.states = []
self.extraFeatures = extraFeatures
self.tmp = {}
def __str__(self):
s = ''
i = 1
for state in self.states:
s = s + 'State ' + str(i) + "\n"
s = s + '-------' + "\n"
s = s + str(state)
s = s + "\n\n"
i = i + 1
return s
def __repr__(self):
return str(self)
def addState(self, state):
self.states.append(state)
def addStates(self, states):
for state in states:
self.states.append(state)
def getStates(self):
return self.states
def getState(self, index):
if len(self.states) <= index:
raise Exception('Index out of orderbook state.')
return self.states[index]
def getDictState(self, index):
if len(self.dictBook) <= index:
raise Exception('Index out of orderbook state.')
return self.dictBook[list(self.dictBook.keys())[index]]
def summary(self):
"""Prints a summary of the characteristics of the order book"""
nrStates = len(self.getStates())
duration = (self.getState(-1).getTimestamp() -
self.getState(0).getTimestamp()).total_seconds()
statesPerSecond = nrStates / duration
i = 0
s = self.getState(0)
priceChanges = []
while i < nrStates - 1:
i = i + 1
s_next = self.getState(i)
if (s_next.getTimestamp() -
s.getTimestamp()).total_seconds() < 1.0:
continue
else:
priceChanges.append(
(s.getTradePrice(), s_next.getTradePrice()))
s = s_next
rateOfPriceChange = np.mean([abs(x[0] - x[1]) for x in priceChanges])
print('Number of states: ' + str(nrStates))
print('Duration: ' + str(duration))
print('States per second: ' + str(statesPerSecond))
print('Change of price per second: ' + str(rateOfPriceChange))
def getOffsetHead(self, offset):
"""The index (from the beginning of the list) of the first state past
the given offset in seconds.
For example, if offset=3 with 10 states availble, whereas for
simplicity every state has 1 second diff, then the resulting index
would be the one marked with 'i':
|x|x|x|i|_|_|_|_|_|_|
As a result, the elements marked with 'x' are not meant to be used.
"""
if offset == 0:
return 0
states = self.getStates()
startState = states[0]
offsetIndex = 0
consumed = 0.0
while (consumed < offset and offsetIndex < len(states) - 1):
offsetIndex = offsetIndex + 1
state = states[offsetIndex]
consumed = (state.getTimestamp() -
startState.getTimestamp()).total_seconds()
if consumed < offset:
raise Exception('Not enough data for offset. Found states for ' +
str(consumed) + ' seconds, required: ' +
str(offset))
return offsetIndex
def getOffsetTail(self, offset):
"""The index (from the end of the list) of the first state past
the given offset in seconds.
For example, if offset=3 with 10 states availble, whereas for
simplicity every state has 1 second diff, then the resulting index
would be the one marked with 'i':
|_|_|_|_|_|_|i|x|x|x|
As a result, the elements marked with 'x' are not meant to be used.
"""
states = self.getStates()
if offset == 0:
return len(states) - 1
offsetIndex = len(states) - 1
startState = states[offsetIndex]
consumed = 0.0
while (consumed < offset and offsetIndex > 0):
offsetIndex = offsetIndex - 1
state = states[offsetIndex]
consumed = (startState.getTimestamp() -
state.getTimestamp()).total_seconds()
if consumed < offset:
raise Exception('Not enough data for offset. Found states for ' +
str(consumed) + ' seconds, required: ' +
str(offset))
return offsetIndex
def getRandomState(self, runtime, min_head=10):
offsetTail = self.tmp.get('offset_tail_' + str(runtime), None)
if offsetTail is None:
offsetTail = self.getOffsetTail(offset=runtime)
self.tmp['offset_tail_' + str(runtime)] = offsetTail
index = random.choice(range(min_head, offsetTail))
return self.getState(index), index
def createArtificial(self, config):
"""Creates an artificial order book
Args:
config (dict): configuration variables
Example:
config = {
'startPrice': 10010.0,
'endPrice': 10000.0, # Optional
'priceFunction': lambda p0, s, samples: p0 + 10 * np.sin(2*np.pi*10 * (s/samples)), # Optional
'levels': 25,
'qtyPosition' : 1.0,
'startTime': datetime.datetime.now(),
'duration': datetime.timedelta(seconds=100),
'interval': datetime.timedelta(seconds=10),
}
"""
startPrice = config['startPrice']
endPrice = config.get('endPrice')
priceFunction = config.get('priceFunction')
levels = config['levels']
qtyPosition = config['qtyPosition']
startTime = config['startTime']
duration = config['duration']
interval = config['interval']
steps = duration / interval
steps = int(steps + 1)
if endPrice is not None:
gradient = (endPrice - startPrice) / (steps - 1)
prices = [startPrice + i * gradient for i in range(steps)]
elif priceFunction is not None:
prices = [
priceFunction(startPrice, i, steps) for i in np.arange(steps)
]
else:
raise Exception('Define \"endPrice\" or \"priceFunction\"')
times = [startTime + i * interval for i in range(steps)]
for i in range(steps):
p = prices[i]
t = times[i]
#bps = 0.0001 * p
bps = 0.1 # 10 cents
asks = [
OrderbookEntry(price=(p + i * bps), qty=qtyPosition)
for i in range(levels)
]
bids = [
OrderbookEntry(price=(p - (i + 1) * bps), qty=qtyPosition)
for i in range(levels)
]
s = OrderbookState(tradePrice=p, timestamp=t)
s.addBuyers(bids)
s.addSellers(asks)
self.addState(s)
self.generateDict()
def generateDict(self):
""" (Re)Generates dictBook.
This is particularly required for artifically created books or books
loaded other than from the events source.
"""
d = {}
for state in self.getStates():
bids = {}
asks = {}
for x in state.getBuyers():
bids[x.getPrice()] = x.getQty()
for x in state.getSellers():
asks[x.getPrice()] = x.getQty()
ts = state.getTimestamp().timestamp()
d[ts] = {'bids': bids, 'asks': asks}
assert (len(d) == len(self.getStates()))
self.dictBook = d
def loadFromFile(self, file):
import csv
with open(file, 'rt') as tsvin:
tsvin = csv.reader(tsvin, delimiter='\t')
for row in tsvin:
p = float(row[1])
vol = float(row[2])
b1 = float(row[3])
b2 = float(row[4])
b3 = float(row[5])
b4 = float(row[6])
b5 = float(row[7])
a1 = float(row[8])
a2 = float(row[9])
a3 = float(row[10])
a4 = float(row[11])
a5 = float(row[12])
bq1 = float(row[13])
bq2 = float(row[14])
bq3 = float(row[15])
bq4 = float(row[16])
bq5 = float(row[17])
aq1 = float(row[18])
aq2 = float(row[19])
aq3 = float(row[20])
aq4 = float(row[21])
aq5 = float(row[22])
dt = parser.parse(row[24]) # trade timestamp as reference
buyers = [
OrderbookEntry(b1, bq1),
OrderbookEntry(b2, bq2),
OrderbookEntry(b3, bq3),
OrderbookEntry(b4, bq4),
OrderbookEntry(b5, bq5)
]
sellers = [
OrderbookEntry(a1, aq1),
OrderbookEntry(a2, aq2),
OrderbookEntry(a3, aq3),
OrderbookEntry(a4, aq4),
OrderbookEntry(a5, aq5)
]
s = OrderbookState(tradePrice=p, timestamp=dt)
s.addBuyers(buyers)
s.addSellers(sellers)
s.setVolume(vol)
# Hand made features
if self.extraFeatures:
mean60 = float(row[26])
vol60 = float(row[27])
std60 = float(row[28])
s.setMarketVar('mean60', mean60)
s.setMarketVar('vol60', vol60)
s.setMarketVar('std60', std60)
self.addState(s)
def loadFromBitfinexFile(self, file):
import csv
import json
with open(file, 'rt') as tsvin:
tsvin = csv.reader(tsvin, delimiter='\t')
for row in tsvin:
priceBid = float(row[1])
priceAsk = float(row[2])
volumeBid = float(row[3])
volumeAsk = float(row[4])
volume = float(row[5])
bids = json.loads(row[6])
asks = json.loads(row[7])
timestamp = parser.parse(row[8])
buyers = [
OrderbookEntry(price=float(x['price']),
qty=float(x['amount'])) for x in bids
]
sellers = [
OrderbookEntry(price=float(x['price']),
qty=float(x['amount'])) for x in asks
]
s = OrderbookState(tradePrice=priceAsk, timestamp=timestamp)
s.addBuyers(buyers)
s.addSellers(sellers)
s.setVolume(volume)
if self.extraFeatures:
s.setMarketVar(key='volumeBid', value=volumeBid)
s.setMarketVar(key='volumeAsk', value=volumeAsk)
self.addState(s)
@staticmethod
def generateDictFromEvents(events_pd):
""" Generates dictionary based order book.
dict :: {timestamp: state}
state :: {'bids': {price: size}, 'asks': {price, size}}
"""
import copy
most_recent_orderbook = {"bids": {}, "asks": {}}
orderbook = {}
for e in events_pd.itertuples():
if e.is_trade:
continue
if e.size == 0.0:
try:
del most_recent_orderbook["bids" if e.is_bid else "asks"][
e.price]
# print('Cancel ' + str(e.price))
except:
# print('Cancel ' + str(e.price) + ' not in recent book')
continue
else:
current_size = most_recent_orderbook["bids" if e.
is_bid else "asks"].get(
e.price, 0.0)
most_recent_orderbook["bids" if e.is_bid else "asks"][
e.price] = current_size + e.size
orderbook[e.ts] = copy.deepcopy(most_recent_orderbook)
return orderbook
def loadFromDict(self, d):
import collections
from datetime import datetime
# skip states until at least 1 bid and 1 ask is available
skip = 0
while True:
head_key = next(iter(d))
head = d[head_key]
if len(head["bids"]) > 0 and len(head["asks"]) > 0:
break
del d[head_key]
skip = skip + 1
print("Skipped dict entries: " + str(skip))
for ts in iter(d.keys()):
state = d[ts]
bids = collections.OrderedDict(
sorted(state["bids"].items(), reverse=True))
asks = collections.OrderedDict(sorted(state["asks"].items()))
buyers = [
OrderbookEntry(price=float(x[0]), qty=float(x[1]))
for x in bids.items()
]
sellers = [
OrderbookEntry(price=float(x[0]), qty=float(x[1]))
for x in asks.items()
]
if len(sellers) > 0:
s = OrderbookState(tradePrice=max(state["asks"].keys()),
timestamp=datetime.fromtimestamp(ts))
s.addBuyers(buyers)
s.addSellers(sellers)
s.setVolume(0.0)
self.addState(s)
#for s in self.getStates():
# assert(s.getBestBid() <= s.getBestAsk())
def loadFromEventsFrame(self, events_pd):
self.dictBook = Orderbook.generateDictFromEvents(events_pd)
self.loadFromDict(self.dictBook)
self.trades = Orderbook.generateTradesFromEvents(events_pd)
@staticmethod
def generateTradesFromEvents(events_pd):
""" Generates dictionary based on historical trades.
dict :: {timestamp: trade}
state :: {'price': float, 'size': float, side: OrderSide}
"""
import copy
trades = {}
for e in events_pd.itertuples():
if e.is_trade:
if e.is_bid:
#side = OrderSide.BUY
side = 0
else:
#side = OrderSide.SELL
side = 1
trades[e.ts] = {'price': e.price, 'size': e.size, 'side': side}
return trades
def loadFromEvents(
self,
file,
cols=["ts", "seq", "size", "price", "is_bid", "is_trade", "ttype"],
clean=50):
print('Attempt to load from cache.')
o = self.cache.get(file + '.class')
if o is not None:
print('Order book in cache. Load...')
self.states = o.states
self.dictBook = o.dictBook
self.trades = o.trades
else:
print('Order book not in cache. Read from file...')
import pandas as pd
events = pd.read_table(file, sep='\t', names=cols, index_col="seq")
self.loadFromEventsFrame(events.sort_index())
# We remove the first few states as they are lacking bids and asks
for i in range(clean):
self.states.pop(0)
del self.dictBook[list(self.dictBook.keys())[0]]
# Store in cache
print('Cache order book')
self.cache.add(file + '.class', self)
def plot(self, show_bidask=False, max_level=-1, show=True):
import matplotlib.pyplot as plt
plt.figure(figsize=(24, 18))
plt.tick_params(axis='both', which='major', labelsize=25)
plt.tick_params(axis='both', which='minor', labelsize=25)
price = [x.getBidAskMid() for x in self.getStates()]
times = [x.getTimestamp() for x in self.getStates()]
plt.plot(times, price)
if show_bidask:
buyer = [
x.getBuyers()[max_level].getPrice() for x in self.getStates()
]
seller = [
x.getSellers()[max_level].getPrice() for x in self.getStates()
]
plt.plot(times, buyer)
plt.plot(times, seller)
if show == True:
plt.show()
else:
return plt
def createFeatures(self):
volumes = np.array([x.getVolume() for x in self.getStates()])
scaler = MinMaxScaler(feature_range=(0, 5))
volumesScaled = scaler.fit_transform(volumes.reshape(-1, 1))
volumesScaled = volumesScaled.flatten().tolist()
volumesRelative = list(map(round, volumesScaled))
i = 0
for state in self.getStates():
state.setMarketVar('volumeRelativeTotal', volumesRelative[i])
i = i + 1
def getBidAskFeature(self,
bids,
asks,
qty=None,
price=True,
size=True,
normalize=False,
levels=20):
"""Creates feature to represent bids and asks.
The prices and sizes of the bids and asks are normalized by the provided
(naturally current) bestAsk and the provided quantity respectively.
Shape: (2, levels, count(features)), whereas features can be [price, size]
[
[
[bid_price bid_size]
[... ... ]
]
[
[ask_price ask_size]
[... ... ]
]
]
"""
assert (price is True or size is True)
def toArray(d):
s = pd.Series(d, name='size')
s.index.name = 'price'
s = s.reset_index()
return np.array(s)
def force_levels(a, n=levels):
"""Shrinks or expands array to n number of records."""
gap = (n - a.shape[0])
if gap > 0:
gapfill = np.zeros((gap, 2))
a = np.vstack((a, gapfill))
return a
elif gap <= 0:
return a[:n]
bids = OrderedDict(sorted(bids.items(), reverse=True))
asks = OrderedDict(sorted(asks.items()))
bids = toArray(bids)
asks = toArray(asks)
if normalize is True:
assert (qty is not None)
bestAsk = np.min(asks[:, 0])
bids = np.column_stack((bids[:, 0] / bestAsk, bids[:, 1] / qty))
asks = np.column_stack((asks[:, 0] / bestAsk, asks[:, 1] / qty))
bidsAsks = np.array([force_levels(bids), force_levels(asks)])
if price is True and size is True:
return bidsAsks
if price is True:
return bidsAsks[:, :, 0]
if size is True:
return bidsAsks[:, :, 1]
def getBidAskFeatures(self,
state_index,
lookback,
qty=None,
price=True,
size=True,
normalize=True,
levels=20):
""" Creates feature to represent bids and asks with a lookback of previous states.
Shape: (2*lookback, levels, count(features))
"""
assert (state_index >= lookback)
state = self.getDictState(state_index)
asks = state['asks']
bids = state['bids']
i = 0
while i < lookback:
state_index = state_index - 1
state = self.getDictState(state_index)
asks = state['asks']
bids = state['bids']
features_next = self.getBidAskFeature(bids=bids,
asks=asks,
qty=qty,
price=price,
size=size,
normalize=normalize,
levels=levels)
if i == 0:
features = np.array(features_next)
else:
features = np.vstack((features, features_next))
i = i + 1
return features
def get_hist_trades(self, ts, lookback=20):
ts -= 3600
acc_trades = {}
for ts_tmp, trade_tmp in sorted(list(self.trades.items()),
reverse=True):
if ts_tmp <= ts:
acc_trades[ts_tmp] = trade_tmp
if len(acc_trades) == lookback:
break
return acc_trades
def getHistTradesFeature(self,
ts,
lookback=20,
normalize=True,
norm_price=None,
norm_size=None):
trades = self.get_hist_trades(ts, lookback=lookback)
trades = list(
map(lambda v: [v['price'], v['size'], v['side']], trades.values()))
arr = np.array(trades)
if normalize:
arr = np.column_stack(
(arr[:, 0] / norm_price, arr[:, 1] / norm_size, arr[:, 2]))
return arr
