def load_trades(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
bound_stmt = self.query_trades_stmt.bind([sub_key.inst_id, from_timestamp, to_timestamp])
result_list = self.session.execute(bound_stmt)
return [Trade(inst_id=r.inst_id, timestamp=r.timestamp, price=r.price, size=r.size) for r in result_list]
def deserialize(data):
if isinstance(data, dict):
if b'__datetime__' in data:
return DateUtils.timestamp_to_datetime(data[b'__datetime__'])
elif b'__date__' in data:
return DateUtils.timestamp_to_date(data[b'__date__'])
# elif '__datetime__' in data:
# return DateUtils.timestamp_to_datetime(data["__datetime__"])
# elif '__date__' in data:
# return DateUtils.timestamp_to_date(data["__date__"])
elif b'@t' in data:
data = data
module = data[b'@p']
cls = data[b'@t']
if (module, cls) not in Serializer.cls_cache:
m = importlib.import_module(module)
if not hasattr(m, cls):
print data
c = getattr(m, cls)
Serializer.cls_cache[(module, cls)] = c
c = Serializer.cls_cache[(module, cls)]
obj = c()
MapSerializer._deserialize_obj(obj, data)
return obj
else:
return {MapSerializer.deserialize(k): MapSerializer.deserialize(v) for k, v in data.iteritems()}
elif isinstance(data, list):
return [MapSerializer.deserialize(i) for i in data]
elif isinstance(data, tuple):
return tuple([MapSerializer.deserialize(i) for i in data])
elif isinstance(data, set):
return set([MapSerializer.deserialize(i) for i in data])
elif isinstance(data, basestring):
return str(data)
return data
def load_trades(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
trade for trade in self.load_all('trades') if self.__matches_data(
trade, sub_key.inst_id, from_timestamp, to_timestamp)
]
def __read_csv(self, sub_keys):
dfs = []
sub_key_range = {sub_key.inst_id: (
DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for
sub_key in sub_keys}
for sub_key in sub_keys:
## TODO support different format, e.g. BAR, Quote, Trade csv files
if isinstance(sub_key.subscription_type,
BarSubscriptionType) and sub_key.subscription_type.bar_type == BarType.Time and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
df = self.read_csv(symbol, '%s/%s.csv' % (self.path, symbol.lower()))
dfs.append(df)
df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def load_market_depths(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
market_depth for market_depth in self.load_all('market_depths')
if self.__matches_data(market_depth, sub_key.inst_id,
from_timestamp, to_timestamp)
]
def load_trades(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [self.serializer.deserialize(data)
for data in self.trades.find({"__slots__.inst_id": sub_key.inst_id,
"__slots__.timestamp": {"$gte": from_timestamp,
"$lt": to_timestamp}
})]
def load_market_depths(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
bound_stmt = self.query_market_depths_stmt.bind([sub_key.inst_id, sub_key.subscription_type.provider_id, from_timestamp, to_timestamp])
result_list = self.session.execute(bound_stmt)
return [MarketDepth(inst_id=r.inst_id, provider_id=r.provider_id, timestamp=r.timestamp,
position=r.position, operation=r.operation, side=r.side, price=r.price, size=r.size) for r
in result_list]
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
bar for bar in self.load_all('bars') if self.__matches_data(
bar, sub_key.inst_id, from_timestamp, to_timestamp)
and bar.type == sub_key.subscription_type.bar_type
and bar.size == sub_key.subscription_type.bar_size
]
def test_real_time_clock_schedule_relative(self):
start = self.realtime_clock.now()
print start
print DateUtils.unixtimemillis_to_datetime(start)
self.realtime_clock.schedule_relative(datetime.timedelta(seconds=1), self.realtime_action)
self.assertEquals([], self.endtime)
time.sleep(1.1)
self.assertEquals(1, len(self.endtime))
self.assertAlmostEqual(1000, self.endtime[0] - start, -2)
def load_quotes(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
bound_stmt = self.query_quotes_stmt.bind([sub_key.inst_id, from_timestamp, to_timestamp])
result_list = self.session.execute(bound_stmt)
return [Quote(inst_id=r.inst_id, timestamp=r.timestamp, bid=r.bid, ask=r.ask, bid_size=r.bid_size,
ask_size=r.ask_size) for r in result_list]
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [self.serializer.deserialize(data)
for data in self.bars.find({"__slots__.inst_id": sub_key.inst_id,
"__slots__.type": sub_key.subscription_type.bar_type,
"__slots__.size": sub_key.subscription_type.bar_size,
"__slots__.timestamp": {"$gte": from_timestamp,
"$lt": to_timestamp}
})]
def test_real_time_clock_schedule_relative(self):
start = self.realtime_clock.now()
print start
print DateUtils.unixtimemillis_to_datetime(start)
self.realtime_clock.schedule_relative(datetime.timedelta(seconds=1),
self.realtime_action)
self.assertEquals([], self.endtime)
time.sleep(1.1)
self.assertEquals(1, len(self.endtime))
self.assertAlmostEqual(1000, self.endtime[0] - start, -2)
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
bound_stmt = self.query_bars_stmt.bind(
[sub_key.inst_id, sub_key.subscription_type.bar_type, sub_key.subscription_type.bar_size,
from_timestamp, to_timestamp])
result_list = self.session.execute(bound_stmt)
return [Bar(inst_id=r.inst_id, type=r.type, size=r.size, begin_time=r.begin_time, timestamp=r.timestamp,
open=r.open, high=r.high, low=r.low, close=r.close, vol=r.vol, adj_close=r.adj_close) for r in
result_list]
def load_market_depths(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
self.serializer.deserialize(data)
for data in self.market_depths.find({
"__slots__.inst_id": sub_key.inst_id,
"__slots__.timestamp": {
"$gte": from_timestamp,
"$lt": to_timestamp
}
})
]
def test_timestamp_conversion(self):
dt = datetime.datetime(year=2000, month=1, day=1, hour=7, minute=30, second=30)
ts = DateUtils.datetime_to_unixtimemillis(dt)
self.assertEqual(946683030000, ts)
dt2 = DateUtils.unixtimemillis_to_datetime(ts)
self.assertEquals(dt, dt2)
dt3 = datetime.datetime.fromtimestamp(0)
ts2 = DateUtils.datetime_to_unixtimemillis(dt3)
dt4 = DateUtils.unixtimemillis_to_datetime(ts2)
self.assertEquals(0, ts2)
self.assertEquals(dt3, dt4)
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [
self.serializer.deserialize(data) for data in self.bars.find(
{
"__slots__.inst_id": sub_key.inst_id,
"__slots__.type": sub_key.subscription_type.bar_type,
"__slots__.size": sub_key.subscription_type.bar_size,
"__slots__.timestamp": {
"$gte": from_timestamp,
"$lt": to_timestamp
}
})
]
def get_current_bar_start_time(timestamp, bar_size):
if bar_size < BarSize.D1:
return int(timestamp / (bar_size * 1000)) * bar_size * 1000
else:
dt = datetime.datetime.fromtimestamp(timestamp / 1000)
return DateUtils.datetime_to_unixtimemillis(
datetime.datetime(year=dt.year, month=dt.month, day=dt.day))
def test_subscribe_bars(self, name, datastore):
start_date = date(2011, 1, 1)
end_date = date(2011, 1, 5)
sub_key = HistDataSubscriptionKey(
inst_id=10,
provider_id=Broker.IB,
subscription_type=BarSubscriptionType(bar_type=BarType.Time,
bar_size=BarSize.D1),
from_date=start_date,
to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1, 5):
persistable = Bar(
timestamp=DateUtils.date_to_unixtimemillis(date_val),
type=BarType.Time,
size=BarSize.D1,
inst_id=10,
open=18 + i,
high=19 + i,
low=17 + i,
close=17.5 + i,
vol=100)
datastore.save_bar(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def test_subscribe_market_depths(self, name, datastore):
start_date = date(2011, 1, 1)
end_date = date(2011, 1, 5)
sub_key = HistDataSubscriptionKey(
inst_id=10,
provider_id=Broker.IB,
subscription_type=MarketDepthSubscriptionType(provider_id='20'),
from_date=start_date,
to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1, 5):
persistable = MarketDepth(
timestamp=DateUtils.date_to_unixtimemillis(date_val),
inst_id=10,
provider_id='20',
position=10 + i,
operation=MDOperation.Insert,
side=MDSide.Ask,
price=10.1 + i,
size=20)
datastore.save_market_depth(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def test_subscribe_trades(self, name, datastore):
start_date = date(2011, 1, 1)
end_date = date(2011, 1, 5)
sub_key = HistDataSubscriptionKey(
inst_id=10,
provider_id=Broker.IB,
subscription_type=TradeSubscriptionType(),
from_date=start_date,
to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1, 5):
persistable = Trade(
timestamp=DateUtils.date_to_unixtimemillis(date_val),
price=20 + i,
size=200 + i,
inst_id=10)
datastore.save_trade(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def on_bar(self, bar):
reval_date = DateUtils.unixtimemillis_to_datetime(bar.timestamp)
futures_expiry_dict = VIXFuture.future_expirydays_calculator(reval_date, self.instruments)
active_futures = {k: v for k, v in futures_expiry_dict.iteritems()
if v > self.exp_date_lb and v < self.exp_date_ub}
active_futures_sorted = OrderedDict(sorted(active_futures.items(), key=lambda y: y[1]))
if bar.inst_id in active_futures_sorted:
logger.debug("id is in active futures" % bar.inst_id)
roll = VIXFuture.daily_roll(bar.adj_close, self.vix_index.now("value"), active_futures_sorted[bar.inst_id])
logger.debug("roll = %s" % roll)
if not self.portfolio.has_position(self.stg_id, bar.inst_id):
threshold = self.get_stg_config_value("short_entry_threshold", 0.02)
if roll > threshold:
logger.debug("Roll > threshold %s" % threshold)
logger.debug("Now send a short order")
self.market_order(inst_id=bar.inst_id, action=OrdAction.SELL, qty=self.qty)
else:
threshold = self.get_stg_config_value("short_exit_threshold", -0.01)
if roll < threshold:
logger.debug("Roll < threshold %s" % threshold)
logger.debug("Now exit")
self.market_order(inst_id=bar.inst_id, action=OrdAction.BUY, qty=self.qty)
def test_real_time_clock_now(self):
ts = DateUtils.datetime_to_unixtimemillis(datetime.datetime.now())
ts2 = self.realtime_clock.now()
self.assertTrue(abs(ts - ts2) <= 10)
time.sleep(2)
ts3 = self.realtime_clock.now()
self.assertAlmostEqual(ts3 - ts2, 2000, -2)
def test_real_time_clock_now(self):
ts = DateUtils.datetime_to_unixtimemillis(datetime.datetime.now())
ts2 = self.realtime_clock.now()
self.assertTrue(abs(ts - ts2) <= 10)
time.sleep(2)
ts3 = self.realtime_clock.now()
self.assertAlmostEqual(ts3 - ts2, 2000, -2)
def _deep_serialize(item, include_slots=True, include_dict=False):
if isinstance(item, Serializable):
return MapSerializer.serialize(item, include_slots, include_dict)
elif isinstance(item, list):
return [MapSerializer._deep_serialize(i) for i in item]
elif isinstance(item, dict):
return {MapSerializer.deserialize(k): MapSerializer._deep_serialize(v) for k, v in item.iteritems()}
elif isinstance(item, tuple):
return tuple([MapSerializer._deep_serialize(i) for i in item])
elif isinstance(item, set):
return set([MapSerializer._deep_serialize(i) for i in item])
elif isinstance(item, datetime.datetime):
return {'__datetime__': DateUtils.datetime_to_timestamp(item)}
elif isinstance(item, datetime.date):
return {'__date__': DateUtils.date_to_timestamp(item)}
else:
return item
def test_real_time_clock_now(self):
s1 = gevent.core.time()
s2 = datetime.datetime.fromtimestamp(s1)
print s1, s2
s3 = self.realtime_clock.now()
s4 = DateUtils.unixtimemillis_to_datetime(s3)
print s3, s4
self.assertAlmostEqual(s1 * 1000, s3, -2)
def test_real_time_clock_schedule_absolute(self):
start = self.realtime_clock.now()
dt = DateUtils.unixtimemillis_to_datetime(start)
abs_time = dt + datetime.timedelta(seconds=1)
self.realtime_clock.schedule_absolute(abs_time, self.realtime_action)
self.assertEquals([], self.endtime)
time.sleep(1.1)
self.assertEquals(1, len(self.endtime))
self.assertAlmostEqual(1000, self.endtime[0] - start, -2)
def test_real_time_clock_now(self):
s1 = gevent.core.time()
s2 = datetime.datetime.fromtimestamp(s1)
print s1, s2
s3 = self.realtime_clock.now()
s4 = DateUtils.unixtimemillis_to_datetime(s3)
print s3, s4
self.assertAlmostEqual(s1 * 1000, s3, -2)
def test_real_time_clock_schedule_absolute(self):
start = self.realtime_clock.now()
dt = DateUtils.unixtimemillis_to_datetime(start)
abs_time = dt + datetime.timedelta(seconds=1)
self.realtime_clock.schedule_absolute(abs_time, self.realtime_action)
self.assertEquals([], self.endtime)
time.sleep(1.1)
self.assertEquals(1, len(self.endtime))
self.assertAlmostEqual(1000, self.endtime[0] - start, -2)
def __load_data(self, sub_keys):
dfs = []
sub_key_range = {
sub_key.inst_id:
(DateUtils.date_to_unixtimemillis(sub_key.from_date),
DateUtils.date_to_unixtimemillis(sub_key.to_date))
for sub_key in sub_keys
}
for sub_key in sub_keys:
if not isinstance(sub_key, HistDataSubscriptionKey):
raise RuntimeError(
"only HistDataSubscriptionKey is supported!")
if isinstance(
sub_key.subscription_type, BarSubscriptionType
) and sub_key.subscription_type.bar_type == BarType.Time: #and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
# df = web.DataReader("F", self.system, sub_key.from_date, sub_key.to_date)
df = self.dict_of_df[symbol]
# df['Symbol'] = symbol
# df['BarSize'] = int(BarSize.M5)
dfs.append(df)
if len(dfs) > 0:
self.df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def process_row(self, index, row):
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=DateUtils.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def process_row(self, index, row):
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=DateUtils.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def test_timestamp_conversion(self):
dt = datetime.datetime(year=2000,
month=1,
day=1,
hour=7,
minute=30,
second=30)
ts = DateUtils.datetime_to_unixtimemillis(dt)
self.assertEqual(946683030000, ts)
dt2 = DateUtils.unixtimemillis_to_datetime(ts)
self.assertEquals(dt, dt2)
dt3 = datetime.datetime.fromtimestamp(0)
ts2 = DateUtils.datetime_to_unixtimemillis(dt3)
dt4 = DateUtils.unixtimemillis_to_datetime(ts2)
self.assertEquals(0, ts2)
self.assertEquals(dt3, dt4)
def _deep_serialize(item, include_slots=True, include_dict=False):
if isinstance(item, Serializable):
return MapSerializer.serialize(item, include_slots, include_dict)
elif isinstance(item, list):
return [MapSerializer._deep_serialize(i) for i in item]
elif isinstance(item, dict):
return {
MapSerializer.deserialize(k): MapSerializer._deep_serialize(v)
for k, v in item.iteritems()
}
elif isinstance(item, tuple):
return tuple([MapSerializer._deep_serialize(i) for i in item])
elif isinstance(item, set):
return set([MapSerializer._deep_serialize(i) for i in item])
elif isinstance(item, datetime.datetime):
return {'__datetime__': DateUtils.datetime_to_timestamp(item)}
elif isinstance(item, datetime.date):
return {'__date__': DateUtils.date_to_timestamp(item)}
else:
return item
def process_row(self, index, row):
logger.debug("[%s] process_row with index %s, symbol %s" % (self.__class__.__name__, index, row['Symbol']))
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=DateUtils.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def expected(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None):
year = year if year >= 0 else BarTest.current_dt.year
month = month if month >= 0 else BarTest.current_dt.month
day = day if day >= 0 else BarTest.current_dt.day
hour = hour if hour >= 0 else BarTest.current_dt.hour
minute = minute if minute >= 0 else BarTest.current_dt.minute
second = second if second >= 0 else BarTest.current_dt.second
microsecond = microsecond if microsecond >= 0 else BarTest.current_dt.microsecond
return DateUtils.datetime_to_unixtimemillis(
datetime.datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second,
microsecond=microsecond))
def process_row(self, index, row):
logger.debug("[%s] process_row with index %s, symbol %s" %
(self.__class__.__name__, index, row['Symbol']))
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=DateUtils.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def __load_data(self, sub_keys):
dfs = []
sub_key_range = {sub_key.inst_id: (
DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for
sub_key in sub_keys}
for sub_key in sub_keys:
if not isinstance(sub_key, HistDataSubscriptionKey):
raise RuntimeError("only HistDataSubscriptionKey is supported!")
if isinstance(sub_key.subscription_type,
BarSubscriptionType) and sub_key.subscription_type.bar_type == BarType.Time: #and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
# df = web.DataReader("F", self.system, sub_key.from_date, sub_key.to_date)
df = self.dict_of_df[symbol]
# df['Symbol'] = symbol
# df['BarSize'] = int(BarSize.M5)
dfs.append(df)
if len(dfs) > 0:
self.df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def deserialize(data):
if isinstance(data, dict):
if b'__datetime__' in data:
return DateUtils.timestamp_to_datetime(data[b'__datetime__'])
elif b'__date__' in data:
return DateUtils.timestamp_to_date(data[b'__date__'])
# elif '__datetime__' in data:
# return DateUtils.timestamp_to_datetime(data["__datetime__"])
# elif '__date__' in data:
# return DateUtils.timestamp_to_date(data["__date__"])
elif b'@t' in data:
data = data
module = data[b'@p']
cls = data[b'@t']
if (module, cls) not in Serializer.cls_cache:
m = importlib.import_module(module)
if not hasattr(m, cls):
print data
c = getattr(m, cls)
Serializer.cls_cache[(module, cls)] = c
c = Serializer.cls_cache[(module, cls)]
obj = c()
MapSerializer._deserialize_obj(obj, data)
return obj
else:
return {
MapSerializer.deserialize(k): MapSerializer.deserialize(v)
for k, v in data.iteritems()
}
elif isinstance(data, list):
return [MapSerializer.deserialize(i) for i in data]
elif isinstance(data, tuple):
return tuple([MapSerializer.deserialize(i) for i in data])
elif isinstance(data, set):
return set([MapSerializer.deserialize(i) for i in data])
elif isinstance(data, basestring):
return str(data)
return data
def test_multi_subscriptions(self, name, datastore):
start_date = date(2011,1,1)
end_date = date(2011,1,5)
sub_key1 = HistDataSubscriptionKey(inst_id=99, provider_id=Broker.IB,
subscription_type=BarSubscriptionType(bar_type=BarType.Time, bar_size=BarSize.D1),
from_date=start_date, to_date=end_date)
sub_key2 = HistDataSubscriptionKey(inst_id=99, provider_id=Broker.IB,
subscription_type=QuoteSubscriptionType(),
from_date=start_date, to_date=end_date)
sub_key3 = HistDataSubscriptionKey(inst_id=99, provider_id=Broker.IB,
subscription_type=TradeSubscriptionType(),
from_date=start_date, to_date=end_date)
expect_val = []
#out of range
persistable = Bar(timestamp=DateUtils.date_to_unixtimemillis(date(2010,12,31)), type=BarType.Time, size=BarSize.D1, inst_id=99, open=18, high=19, low=17, close=17.5, vol=100)
datastore.save_bar(persistable)
persistable = Bar(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,1)), type=BarType.Time, size=BarSize.D1, inst_id=99, open=28, high=29, low=27, close=27.5, vol=100)
datastore.save_bar(persistable)
expect_val.append(persistable)
persistable = Trade(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,2)), price=20, size=200, inst_id=99)
datastore.save_trade(persistable)
expect_val.append(persistable)
persistable = Trade(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,3)), price=30, size=200, inst_id=99)
datastore.save_trade(persistable)
expect_val.append(persistable)
# not same instrument
persistable = Quote(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,3)), bid=18, ask=19, bid_size=200, ask_size=500, inst_id=11)
datastore.save_quote(persistable)
persistable = Quote(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,4)), bid=18, ask=19, bid_size=200, ask_size=500, inst_id=99)
datastore.save_quote(persistable)
expect_val.append(persistable)
# out of range
persistable = Quote(timestamp=DateUtils.date_to_unixtimemillis(date(2011,1,5)), bid=28, ask=29, bid_size=200, ask_size=500, inst_id=99)
datastore.save_quote(persistable)
actual_val = datastore.load_mktdata(sub_key1, sub_key2, sub_key3)
self.assertEqual(expect_val, actual_val)
def test_subscribe_trades(self, name, datastore):
start_date = date(2011,1,1)
end_date = date(2011,1,5)
sub_key = HistDataSubscriptionKey(inst_id=10, provider_id=Broker.IB,
subscription_type=TradeSubscriptionType(),
from_date=start_date, to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1,5):
persistable = Trade(timestamp=DateUtils.date_to_unixtimemillis(date_val), price=20+i, size=200+i, inst_id=10)
datastore.save_trade(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def test_subscribe_bars(self, name, datastore):
start_date = date(2011,1,1)
end_date = date(2011,1,5)
sub_key = HistDataSubscriptionKey(inst_id=10, provider_id=Broker.IB,
subscription_type=BarSubscriptionType(bar_type=BarType.Time, bar_size=BarSize.D1),
from_date=start_date, to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1,5):
persistable = Bar(timestamp=DateUtils.date_to_unixtimemillis(date_val), type=BarType.Time, size=BarSize.D1, inst_id=10, open=18 + i, high=19 + i, low=17 + i , close=17.5 + i, vol=100)
datastore.save_bar(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def test_subscribe_market_depths(self, name, datastore):
start_date = date(2011,1,1)
end_date = date(2011,1,5)
sub_key = HistDataSubscriptionKey(inst_id=10, provider_id=Broker.IB,
subscription_type=MarketDepthSubscriptionType(provider_id='20'),
from_date=start_date, to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1,5):
persistable = MarketDepth(timestamp=DateUtils.date_to_unixtimemillis(date_val), inst_id=10, provider_id='20', position=10+i,
operation=MDOperation.Insert, side=MDSide.Ask,
price=10.1+i, size=20)
datastore.save_market_depth(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def on_bar(self, bar):
reval_date = DateUtils.unixtimemillis_to_datetime(bar.timestamp)
futures_expiry_dict = VIXFuture.future_expirydays_calculator(
reval_date, self.instruments)
active_futures = {
k: v
for k, v in futures_expiry_dict.iteritems()
if v > self.exp_date_lb and v < self.exp_date_ub
}
active_futures_sorted = OrderedDict(
sorted(active_futures.items(), key=lambda y: y[1]))
if bar.inst_id in active_futures_sorted:
logger.debug("id is in active futures" % bar.inst_id)
roll = VIXFuture.daily_roll(bar.adj_close,
self.vix_index.now("value"),
active_futures_sorted[bar.inst_id])
logger.debug("roll = %s" % roll)
if not self.portfolio.has_position(self.stg_id, bar.inst_id):
threshold = self.get_stg_config_value("short_entry_threshold",
0.02)
if roll > threshold:
logger.debug("Roll > threshold %s" % threshold)
logger.debug("Now send a short order")
self.market_order(inst_id=bar.inst_id,
action=OrdAction.SELL,
qty=self.qty)
else:
threshold = self.get_stg_config_value("short_exit_threshold",
-0.01)
if roll < threshold:
logger.debug("Roll < threshold %s" % threshold)
logger.debug("Now exit")
self.market_order(inst_id=bar.inst_id,
action=OrdAction.BUY,
qty=self.qty)
def expected(self,
year=None,
month=None,
day=None,
hour=None,
minute=None,
second=None,
microsecond=None):
year = year if year >= 0 else BarTest.current_dt.year
month = month if month >= 0 else BarTest.current_dt.month
day = day if day >= 0 else BarTest.current_dt.day
hour = hour if hour >= 0 else BarTest.current_dt.hour
minute = minute if minute >= 0 else BarTest.current_dt.minute
second = second if second >= 0 else BarTest.current_dt.second
microsecond = microsecond if microsecond >= 0 else BarTest.current_dt.microsecond
return DateUtils.datetime_to_unixtimemillis(
datetime.datetime(year=year,
month=month,
day=day,
hour=hour,
minute=minute,
second=second,
microsecond=microsecond))
class BarTest(TestCase):
current_dt = datetime.datetime(year=2016,
month=8,
day=1,
hour=6,
minute=3,
second=4)
current_ts = DateUtils.datetime_to_unixtimemillis(current_dt)
def ts(self, func, size):
return func(BarTest.current_ts, size)
def expected(self,
year=None,
month=None,
day=None,
hour=None,
minute=None,
second=None,
microsecond=None):
year = year if year >= 0 else BarTest.current_dt.year
month = month if month >= 0 else BarTest.current_dt.month
day = day if day >= 0 else BarTest.current_dt.day
hour = hour if hour >= 0 else BarTest.current_dt.hour
minute = minute if minute >= 0 else BarTest.current_dt.minute
second = second if second >= 0 else BarTest.current_dt.second
microsecond = microsecond if microsecond >= 0 else BarTest.current_dt.microsecond
return DateUtils.datetime_to_unixtimemillis(
datetime.datetime(year=year,
month=month,
day=day,
hour=hour,
minute=minute,
second=second,
microsecond=microsecond))
def test_current_bar_start_time(self):
func = Bar.get_current_bar_start_time
self.assertEqual(self.expected(second=4), self.ts(func, BarSize.S1))
self.assertEqual(self.expected(second=0, microsecond=0),
self.ts(func, BarSize.S5))
self.assertEqual(self.expected(second=0, microsecond=0),
self.ts(func, BarSize.S15))
self.assertEqual(self.expected(second=0, microsecond=0),
self.ts(func, BarSize.S30))
self.assertEqual(self.expected(second=0, microsecond=0),
self.ts(func, BarSize.M1))
self.assertEqual(self.expected(minute=0, second=0, microsecond=0),
self.ts(func, BarSize.M5))
self.assertEqual(self.expected(minute=0, second=0, microsecond=0),
self.ts(func, BarSize.M15))
self.assertEqual(self.expected(minute=0, second=0, microsecond=0),
self.ts(func, BarSize.M30))
self.assertEqual(self.expected(minute=0, second=0, microsecond=0),
self.ts(func, BarSize.H1))
self.assertEqual(
self.expected(hour=0, minute=0, second=0, microsecond=0),
self.ts(func, BarSize.D1))
def test_current_bar_end_time(self):
func = Bar.get_current_bar_end_time
self.assertEqual(self.expected(second=4, microsecond=999999),
self.ts(func, BarSize.S1))
self.assertEqual(self.expected(second=4, microsecond=999999),
self.ts(func, BarSize.S5))
self.assertEqual(self.expected(second=14, microsecond=999999),
self.ts(func, BarSize.S15))
self.assertEqual(self.expected(second=29, microsecond=999999),
self.ts(func, BarSize.S30))
self.assertEqual(self.expected(second=59, microsecond=999999),
self.ts(func, BarSize.M1))
self.assertEqual(
self.expected(minute=4, second=59, microsecond=999999),
self.ts(func, BarSize.M5))
self.assertEqual(
self.expected(minute=14, second=59, microsecond=999999),
self.ts(func, BarSize.M15))
self.assertEqual(
self.expected(minute=29, second=59, microsecond=999999),
self.ts(func, BarSize.M30))
self.assertEqual(
self.expected(minute=59, second=59, microsecond=999999),
self.ts(func, BarSize.H1))
self.assertEqual(
self.expected(hour=23, minute=59, second=59, microsecond=999999),
self.ts(func, BarSize.D1))
def test_next_bar_start_time(self):
func = Bar.get_next_bar_start_time
self.assertEqual(self.expected(second=5, microsecond=0),
self.ts(func, BarSize.S1))
self.assertEqual(self.expected(second=5, microsecond=0),
self.ts(func, BarSize.S5))
self.assertEqual(self.expected(second=15, microsecond=0),
self.ts(func, BarSize.S15))
self.assertEqual(self.expected(second=30, microsecond=0),
self.ts(func, BarSize.S30))
self.assertEqual(self.expected(minute=4, second=0, microsecond=0),
self.ts(func, BarSize.M1))
self.assertEqual(self.expected(minute=5, second=0, microsecond=0),
self.ts(func, BarSize.M5))
self.assertEqual(self.expected(minute=15, second=0, microsecond=0),
self.ts(func, BarSize.M15))
self.assertEqual(self.expected(minute=30, second=0, microsecond=0),
self.ts(func, BarSize.M30))
self.assertEqual(
self.expected(hour=7, minute=0, second=0, microsecond=0),
self.ts(func, BarSize.H1))
self.assertEqual(
self.expected(day=2, hour=0, minute=0, second=0, microsecond=0),
self.ts(func, BarSize.D1))
dt = DateUtils.unixtimemillis_to_datetime(start)
abs_time = dt + datetime.timedelta(seconds=1)
self.realtime_clock.schedule_absolute(abs_time, self.realtime_action)
self.assertEquals([], self.endtime)
time.sleep(1.1)
self.assertEquals(1, len(self.endtime))
self.assertAlmostEqual(1000, self.endtime[0] - start, -2)
def test_real_time_clock_schedule_relative(self):
start = self.realtime_clock.now()
print start
print DateUtils.unixtimemillis_to_datetime(start)
self.realtime_clock.schedule_relative(datetime.timedelta(seconds=1), self.realtime_action)
self.assertEquals([], self.endtime)
time.sleep(1.1)
self.assertEquals(1, len(self.endtime))
self.assertAlmostEqual(1000, self.endtime[0] - start, -2)
def test_real_time_clock_now(self):
s1 = gevent.core.time()
s2 = datetime.datetime.fromtimestamp(s1)
print s1, s2
s3 = self.realtime_clock.now()
s4 = DateUtils.unixtimemillis_to_datetime(s3)
print s3, s4
self.assertAlmostEqual(s1 * 1000, s3, -2)
print DateUtils.unixtimemillis_to_datetime(946683030000)
print DateUtils.unixtimemillis_to_datetime(946711830000)
def get_current_bar_start_time(timestamp, bar_size):
if bar_size < BarSize.D1:
return int(timestamp / (bar_size * 1000)) * bar_size * 1000
else:
dt = datetime.datetime.fromtimestamp(timestamp / 1000)
return DateUtils.datetime_to_unixtimemillis(datetime.datetime(year=dt.year, month=dt.month, day=dt.day))
def update_time(self, timestamp):
self.__current_timestamp_mills = timestamp
self.scheduler.advance_to(DateUtils.unixtimemillis_to_datetime(timestamp))
def schedule_absolute(self, datetime, action, state=None):
if isinstance(datetime, (long, int)):
datetime = DateUtils.unixtimemillis_to_datetime(datetime)
self.scheduler.schedule_absolute(datetime, action, state)
abs_time = dt + datetime.timedelta(seconds=1)
self.realtime_clock.schedule_absolute(abs_time, self.realtime_action)
self.assertEquals([], self.endtime)
time.sleep(1.1)
self.assertEquals(1, len(self.endtime))
self.assertAlmostEqual(1000, self.endtime[0] - start, -2)
def test_real_time_clock_schedule_relative(self):
start = self.realtime_clock.now()
print start
print DateUtils.unixtimemillis_to_datetime(start)
self.realtime_clock.schedule_relative(datetime.timedelta(seconds=1),
self.realtime_action)
self.assertEquals([], self.endtime)
time.sleep(1.1)
self.assertEquals(1, len(self.endtime))
self.assertAlmostEqual(1000, self.endtime[0] - start, -2)
def test_real_time_clock_now(self):
s1 = gevent.core.time()
s2 = datetime.datetime.fromtimestamp(s1)
print s1, s2
s3 = self.realtime_clock.now()
s4 = DateUtils.unixtimemillis_to_datetime(s3)
print s3, s4
self.assertAlmostEqual(s1 * 1000, s3, -2)
print DateUtils.unixtimemillis_to_datetime(946683030000)
print DateUtils.unixtimemillis_to_datetime(946711830000)
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [bar for bar in self.load_all('bars') if self.__matches_data(bar, sub_key.inst_id, from_timestamp,
to_timestamp) and bar.type == sub_key.subscription_type.bar_type and bar.size == sub_key.subscription_type.bar_size]
def load_market_depths(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [market_depth for market_depth in self.load_all('market_depths') if
self.__matches_data(market_depth, sub_key.inst_id, from_timestamp, to_timestamp)]
def load_trades(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
return [trade for trade in self.load_all('trades') if
self.__matches_data(trade, sub_key.inst_id, from_timestamp, to_timestamp)]