def load_trades(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
trade for trade in self.load_all('trades') if self.__matches_data(
trade, sub_key.inst_id, from_timestamp, to_timestamp)
]
def load_trades(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
bound_stmt = self.query_trades_stmt.bind([sub_key.inst_id, from_timestamp, to_timestamp])
result_list = self.session.execute(bound_stmt)
return [Trade(inst_id=r.inst_id, timestamp=r.timestamp, price=r.price, size=r.size) for r in result_list]
def __read_csv(self, sub_keys):
dfs = []
sub_key_range = {sub_key.inst_id: (
DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for
sub_key in sub_keys}
for sub_key in sub_keys:
## TODO support different format, e.g. BAR, Quote, Trade csv files
if isinstance(sub_key.subscription_type,
BarSubscriptionType) and sub_key.subscription_type.bar_type == BarType.Time and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
df = self.read_csv(symbol, '%s/%s.csv' % (self.path, symbol.lower()))
dfs.append(df)
df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def load_trades(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [self.serializer.deserialize(data)
for data in self.trades.find({"__slots__.inst_id": sub_key.inst_id,
"__slots__.timestamp": {"$gte": from_timestamp,
"$lt": to_timestamp}
})]
def load_market_depths(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
market_depth for market_depth in self.load_all('market_depths')
if self.__matches_data(market_depth, sub_key.inst_id,
from_timestamp, to_timestamp)
]
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
bar for bar in self.load_all('bars') if self.__matches_data(
bar, sub_key.inst_id, from_timestamp, to_timestamp)
and bar.type == sub_key.subscription_type.bar_type
and bar.size == sub_key.subscription_type.bar_size
]
def load_quotes(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
bound_stmt = self.query_quotes_stmt.bind([sub_key.inst_id, from_timestamp, to_timestamp])
result_list = self.session.execute(bound_stmt)
return [Quote(inst_id=r.inst_id, timestamp=r.timestamp, bid=r.bid, ask=r.ask, bid_size=r.bid_size,
ask_size=r.ask_size) for r in result_list]
def load_market_depths(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
bound_stmt = self.query_market_depths_stmt.bind([sub_key.inst_id, sub_key.subscription_type.provider_id, from_timestamp, to_timestamp])
result_list = self.session.execute(bound_stmt)
return [MarketDepth(inst_id=r.inst_id, provider_id=r.provider_id, timestamp=r.timestamp,
position=r.position, operation=r.operation, side=r.side, price=r.price, size=r.size) for r
in result_list]
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [self.serializer.deserialize(data)
for data in self.bars.find({"__slots__.inst_id": sub_key.inst_id,
"__slots__.type": sub_key.subscription_type.bar_type,
"__slots__.size": sub_key.subscription_type.bar_size,
"__slots__.timestamp": {"$gte": from_timestamp,
"$lt": to_timestamp}
})]
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
bound_stmt = self.query_bars_stmt.bind(
[sub_key.inst_id, sub_key.subscription_type.bar_type, sub_key.subscription_type.bar_size,
from_timestamp, to_timestamp])
result_list = self.session.execute(bound_stmt)
return [Bar(inst_id=r.inst_id, type=r.type, size=r.size, begin_time=r.begin_time, timestamp=r.timestamp,
open=r.open, high=r.high, low=r.low, close=r.close, vol=r.vol, adj_close=r.adj_close) for r in
result_list]
def load_market_depths(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
self.serializer.deserialize(data)
for data in self.market_depths.find({
"__slots__.inst_id": sub_key.inst_id,
"__slots__.timestamp": {
"$gte": from_timestamp,
"$lt": to_timestamp
}
})
]
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
self.serializer.deserialize(data) for data in self.bars.find(
{
"__slots__.inst_id": sub_key.inst_id,
"__slots__.type": sub_key.subscription_type.bar_type,
"__slots__.size": sub_key.subscription_type.bar_size,
"__slots__.timestamp": {
"$gte": from_timestamp,
"$lt": to_timestamp
}
})
]
def test_subscribe_trades(self, name, datastore):
start_date = date(2011, 1, 1)
end_date = date(2011, 1, 5)
sub_key = HistDataSubscriptionKey(
inst_id=10,
provider_id=Broker.IB,
subscription_type=TradeSubscriptionType(),
from_date=start_date,
to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1, 5):
persistable = Trade(
timestamp=DateUtils.date_to_unixtimemillis(date_val),
price=20 + i,
size=200 + i,
inst_id=10)
datastore.save_trade(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def test_subscribe_market_depths(self, name, datastore):
start_date = date(2011, 1, 1)
end_date = date(2011, 1, 5)
sub_key = HistDataSubscriptionKey(
inst_id=10,
provider_id=Broker.IB,
subscription_type=MarketDepthSubscriptionType(provider_id='20'),
from_date=start_date,
to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1, 5):
persistable = MarketDepth(
timestamp=DateUtils.date_to_unixtimemillis(date_val),
inst_id=10,
provider_id='20',
position=10 + i,
operation=MDOperation.Insert,
side=MDSide.Ask,
price=10.1 + i,
size=20)
datastore.save_market_depth(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def test_subscribe_bars(self, name, datastore):
start_date = date(2011, 1, 1)
end_date = date(2011, 1, 5)
sub_key = HistDataSubscriptionKey(
inst_id=10,
provider_id=Broker.IB,
subscription_type=BarSubscriptionType(bar_type=BarType.Time,
bar_size=BarSize.D1),
from_date=start_date,
to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1, 5):
persistable = Bar(
timestamp=DateUtils.date_to_unixtimemillis(date_val),
type=BarType.Time,
size=BarSize.D1,
inst_id=10,
open=18 + i,
high=19 + i,
low=17 + i,
close=17.5 + i,
vol=100)
datastore.save_bar(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def __load_data(self, sub_keys):
dfs = []
sub_key_range = {
sub_key.inst_id:
(DateUtils.date_to_unixtimemillis(sub_key.from_date),
DateUtils.date_to_unixtimemillis(sub_key.to_date))
for sub_key in sub_keys
}
for sub_key in sub_keys:
if not isinstance(sub_key, HistDataSubscriptionKey):
raise RuntimeError(
"only HistDataSubscriptionKey is supported!")
if isinstance(
sub_key.subscription_type, BarSubscriptionType
) and sub_key.subscription_type.bar_type == BarType.Time: #and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
# df = web.DataReader("F", self.system, sub_key.from_date, sub_key.to_date)
df = self.dict_of_df[symbol]
# df['Symbol'] = symbol
# df['BarSize'] = int(BarSize.M5)
dfs.append(df)
if len(dfs) > 0:
self.df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def __load_data(self, sub_keys):
dfs = []
sub_key_range = {sub_key.inst_id: (
DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for
sub_key in sub_keys}
for sub_key in sub_keys:
if not isinstance(sub_key, HistDataSubscriptionKey):
raise RuntimeError("only HistDataSubscriptionKey is supported!")
if isinstance(sub_key.subscription_type,
BarSubscriptionType) and sub_key.subscription_type.bar_type == BarType.Time: #and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
# df = web.DataReader("F", self.system, sub_key.from_date, sub_key.to_date)
df = self.dict_of_df[symbol]
# df['Symbol'] = symbol
# df['BarSize'] = int(BarSize.M5)
dfs.append(df)
if len(dfs) > 0:
self.df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def test_multi_subscriptions(self, name, datastore):
start_date = date(2011,1,1)
end_date = date(2011,1,5)
sub_key1 = HistDataSubscriptionKey(inst_id=99, provider_id=Broker.IB,
subscription_type=BarSubscriptionType(bar_type=BarType.Time, bar_size=BarSize.D1),
from_date=start_date, to_date=end_date)
sub_key2 = HistDataSubscriptionKey(inst_id=99, provider_id=Broker.IB,
subscription_type=QuoteSubscriptionType(),
from_date=start_date, to_date=end_date)
sub_key3 = HistDataSubscriptionKey(inst_id=99, provider_id=Broker.IB,
subscription_type=TradeSubscriptionType(),
from_date=start_date, to_date=end_date)
expect_val = []
#out of range
persistable = Bar(timestamp=DateUtils.date_to_unixtimemillis(date(2010,12,31)), type=BarType.Time, size=BarSize.D1, inst_id=99, open=18, high=19, low=17, close=17.5, vol=100)
datastore.save_bar(persistable)
persistable = Bar(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,1)), type=BarType.Time, size=BarSize.D1, inst_id=99, open=28, high=29, low=27, close=27.5, vol=100)
datastore.save_bar(persistable)
expect_val.append(persistable)
persistable = Trade(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,2)), price=20, size=200, inst_id=99)
datastore.save_trade(persistable)
expect_val.append(persistable)
persistable = Trade(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,3)), price=30, size=200, inst_id=99)
datastore.save_trade(persistable)
expect_val.append(persistable)
# not same instrument
persistable = Quote(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,3)), bid=18, ask=19, bid_size=200, ask_size=500, inst_id=11)
datastore.save_quote(persistable)
persistable = Quote(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,4)), bid=18, ask=19, bid_size=200, ask_size=500, inst_id=99)
datastore.save_quote(persistable)
expect_val.append(persistable)
# out of range
persistable = Quote(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,5)), bid=28, ask=29, bid_size=200, ask_size=500, inst_id=99)
datastore.save_quote(persistable)
actual_val = datastore.load_mktdata(sub_key1, sub_key2, sub_key3)
self.assertEqual(expect_val, actual_val)
def test_subscribe_bars(self, name, datastore):
start_date = date(2011,1,1)
end_date = date(2011,1,5)
sub_key = HistDataSubscriptionKey(inst_id=10, provider_id=Broker.IB,
subscription_type=BarSubscriptionType(bar_type=BarType.Time, bar_size=BarSize.D1),
from_date=start_date, to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1,5):
persistable = Bar(timestamp=DateUtils.date_to_unixtimemillis(date_val), type=BarType.Time, size=BarSize.D1, inst_id=10, open=18 + i, high=19 + i, low=17 + i , close=17.5 + i, vol=100)
datastore.save_bar(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def test_subscribe_trades(self, name, datastore):
start_date = date(2011,1,1)
end_date = date(2011,1,5)
sub_key = HistDataSubscriptionKey(inst_id=10, provider_id=Broker.IB,
subscription_type=TradeSubscriptionType(),
from_date=start_date, to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1,5):
persistable = Trade(timestamp=DateUtils.date_to_unixtimemillis(date_val), price=20+i, size=200+i, inst_id=10)
datastore.save_trade(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def test_subscribe_market_depths(self, name, datastore):
start_date = date(2011,1,1)
end_date = date(2011,1,5)
sub_key = HistDataSubscriptionKey(inst_id=10, provider_id=Broker.IB,
subscription_type=MarketDepthSubscriptionType(provider_id='20'),
from_date=start_date, to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1,5):
persistable = MarketDepth(timestamp=DateUtils.date_to_unixtimemillis(date_val), inst_id=10, provider_id='20', position=10+i,
operation=MDOperation.Insert, side=MDSide.Ask,
price=10.1+i, size=20)
datastore.save_market_depth(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def load_market_depths(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [market_depth for market_depth in self.load_all('market_depths') if
self.__matches_data(market_depth, sub_key.inst_id, from_timestamp, to_timestamp)]
def load_trades(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [trade for trade in self.load_all('trades') if
self.__matches_data(trade, sub_key.inst_id, from_timestamp, to_timestamp)]
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [bar for bar in self.load_all('bars') if self.__matches_data(bar, sub_key.inst_id, from_timestamp,
to_timestamp) and bar.type == sub_key.subscription_type.bar_type and bar.size == sub_key.subscription_type.bar_size]
def test_multi_subscriptions(self, name, datastore):
start_date = date(2011, 1, 1)
end_date = date(2011, 1, 5)
sub_key1 = HistDataSubscriptionKey(
inst_id=99,
provider_id=Broker.IB,
subscription_type=BarSubscriptionType(bar_type=BarType.Time,
bar_size=BarSize.D1),
from_date=start_date,
to_date=end_date)
sub_key2 = HistDataSubscriptionKey(
inst_id=99,
provider_id=Broker.IB,
subscription_type=QuoteSubscriptionType(),
from_date=start_date,
to_date=end_date)
sub_key3 = HistDataSubscriptionKey(
inst_id=99,
provider_id=Broker.IB,
subscription_type=TradeSubscriptionType(),
from_date=start_date,
to_date=end_date)
expect_val = []
#out of range
persistable = Bar(timestamp=DateUtils.date_to_unixtimemillis(
date(2010, 12, 31)),
type=BarType.Time,
size=BarSize.D1,
inst_id=99,
open=18,
high=19,
low=17,
close=17.5,
vol=100)
datastore.save_bar(persistable)
persistable = Bar(timestamp=DateUtils.date_to_unixtimemillis(
date(2011, 1, 1)),
type=BarType.Time,
size=BarSize.D1,
inst_id=99,
open=28,
high=29,
low=27,
close=27.5,
vol=100)
datastore.save_bar(persistable)
expect_val.append(persistable)
persistable = Trade(timestamp=DateUtils.date_to_unixtimemillis(
date(2011, 1, 2)),
price=20,
size=200,
inst_id=99)
datastore.save_trade(persistable)
expect_val.append(persistable)
persistable = Trade(timestamp=DateUtils.date_to_unixtimemillis(
date(2011, 1, 3)),
price=30,
size=200,
inst_id=99)
datastore.save_trade(persistable)
expect_val.append(persistable)
# not same instrument
persistable = Quote(timestamp=DateUtils.date_to_unixtimemillis(
date(2011, 1, 3)),
bid=18,
ask=19,
bid_size=200,
ask_size=500,
inst_id=11)
datastore.save_quote(persistable)
persistable = Quote(timestamp=DateUtils.date_to_unixtimemillis(
date(2011, 1, 4)),
bid=18,
ask=19,
bid_size=200,
ask_size=500,
inst_id=99)
datastore.save_quote(persistable)
expect_val.append(persistable)
# out of range
persistable = Quote(timestamp=DateUtils.date_to_unixtimemillis(
date(2011, 1, 5)),
bid=28,
ask=29,
bid_size=200,
ask_size=500,
inst_id=99)
datastore.save_quote(persistable)
actual_val = datastore.load_mktdata(sub_key1, sub_key2, sub_key3)
self.assertEqual(expect_val, actual_val)
