def set_value(self, timestamp, open, high, low, close, size):
if timestamp is None or close is None:
logger.warning("[%s] ignore update timestamp=%s, open=%s, high=%s, low=%s, close=%s, size=%s" % (
self.__class__.__name__, timestamp, open, high, low, close, size))
return
if self.__new_bar:
if self.__output_bar_type == BarType.Time:
self.__start_time = Bar.get_current_bar_start_time(timestamp, self.__output_size)
self.__end_time = Bar.get_current_bar_end_time(timestamp, self.__output_size)
else:
self.__start_time = timestamp
self.__open = open
self.__low = low
self.__high = high
self.__new_bar = False
else:
if high > self.__high:
self.__high = high
if low < self.__low:
self.__low = low
self.__count += 1
self.__close = close
self.__volume += size
def test_time_bar_from_ask(self):
agg = BarAggregator(data_bus=self.event_bus, clock=self.simluation_clock, inst_id=1, input=self.input,
input_type=BarInputType.Ask)
agg.start()
self.assertEqual(0, len(self.event_bus.items))
self.time += 10000
t = Quote(timestamp=self.time, inst_id=1, ask=30, ask_size=100)
self.update(self.input, t)
self.assertEqual(1, agg.count())
self.assertTrue(len(self.event_bus.items) == 0)
self.time += 60000
t = Quote(timestamp=self.time, inst_id=1, ask=70, ask_size=300)
self.update(self.input, t)
items = self.event_bus.items
self.assertEqual(1, len(items))
self.assertEqual(1, agg.count())
self.assertEqual(
Bar(inst_id=1, begin_time=9000000000, timestamp=9000059999, type=1, size=60, open=30, high=30, low=30,
close=30, vol=100, adj_close=0), items[0])
self.event_bus.reset()
self.time += 49999
t = Quote(timestamp=self.time, inst_id=1, ask=20, ask_size=100)
self.update(self.input, t)
items = self.event_bus.items
self.assertEqual(1, len(items))
self.assertEqual(0, agg.count())
self.assertEqual(
Bar(inst_id=1, begin_time=9000060000, timestamp=9000119999, type=1, size=60, open=70, high=70, low=20,
close=20, vol=400, adj_close=0), items[0])
def set_value(self, timestamp, open, high, low, close, size):
if timestamp is None or close is None:
logger.warning(
"[%s] ignore update timestamp=%s, open=%s, high=%s, low=%s, close=%s, size=%s"
% (self.__class__.__name__, timestamp, open, high, low, close,
size))
return
if self.__new_bar:
if self.__output_bar_type == BarType.Time:
self.__start_time = Bar.get_current_bar_start_time(
timestamp, self.__output_size)
self.__end_time = Bar.get_current_bar_end_time(
timestamp, self.__output_size)
else:
self.__start_time = timestamp
self.__open = open
self.__low = low
self.__high = high
self.__new_bar = False
else:
if high > self.__high:
self.__high = high
if low < self.__low:
self.__low = low
self.__count += 1
self.__close = close
self.__volume += size
def get_latest_price(self):
price = self.inst_data_mgr.get_latest_price(1)
self.assertIsNone(price)
bar1 = Bar(inst_id=1, open=20, high=21, low=19, close=20.5)
self.inst_data_mgr.on_bar(bar1)
price = self.inst_data_mgr.get_latest_price(1)
self.assertEqual(20.5, price)
bar1 = Bar(inst_id=1,
open=20,
high=21,
low=19,
close=20.5,
adj_close=22)
self.inst_data_mgr.on_bar(bar1)
price = self.inst_data_mgr.get_latest_price(1)
self.assertEqual(22, price)
quote1 = Quote(inst_id=1, bid=18, ask=19, bid_size=200, ask_size=500)
self.inst_data_mgr.on_quote(quote1)
price = self.inst_data_mgr.get_latest_price(1)
self.assertEqual(18.5, price)
trade1 = Trade(inst_id=1, price=20, size=200)
self.inst_data_mgr.on_bar(trade1)
price = self.inst_data_mgr.get_latest_price(1)
self.assertEqual(20, price)
def test_on_limit_order_immediate_fill(self):
bar1 = Bar(inst_id=1, open=20, high=21, low=19, close=20.5, vol=1000)
bar2 = Bar(inst_id=1, open=16, high=18, low=15, close=17, vol=1000)
self.app_context.inst_data_mgr.on_bar(bar2)
orders = self.simulator._get_orders()
self.assertEqual(0, len(orders))
order1 = NewOrderRequest(cl_id='test',
cl_ord_id=1,
inst_id=1,
action=OrdAction.BUY,
type=OrdType.LIMIT,
qty=1000,
limit_price=18.5)
self.simulator.on_new_ord_req(order1)
orders = self.simulator._get_orders()
self.assertEqual(1, len(orders))
self.assertEqual(1, len(orders[order1.inst_id]))
self.assertEqual(0, len(orders[order1.inst_id][order1.cl_id]))
self.assertEqual(2, len(self.exec_handler.exec_reports))
exec_report = self.exec_handler.exec_reports[0]
self.assert_exec_report(exec_report, order1.cl_id, order1.cl_ord_id, 0,
0, OrdStatus.SUBMITTED)
exec_report = self.exec_handler.exec_reports[1]
self.assert_exec_report(exec_report, order1.cl_id, order1.cl_ord_id,
1000, 18.5, OrdStatus.FILLED)
def test_stop_order_handler(self):
handler = StopOrderHandler(self.config)
bar1 = Bar(inst_id=1, open=20, high=21, low=19, close=20.5, vol=1000)
bar2 = Bar(inst_id=1, open=16, high=18, low=15, close=17, vol=1000)
# BUY
order1 = NewOrderRequest(cl_id='test', cl_ord_id=1, inst_id=1, action=OrdAction.BUY, type=OrdType.STOP,
qty=1000, stop_price=18.5)
fill_info = handler.process_w_price_qty(order1, 18, 1000)
self.assertFalse(handler.stop_limit_ready(order1.cl_id, order1.cl_ord_id))
self.assertEquals(None, fill_info)
fill_info = handler.process_w_price_qty(order1, 19, 1000)
self.assertTrue(handler.stop_limit_ready(order1.cl_id, order1.cl_ord_id))
self.assertEquals(19, fill_info.fill_price)
self.assertEquals(1000, fill_info.fill_qty)
# BUY with bar
order2 = NewOrderRequest(cl_id='test', cl_ord_id=2, inst_id=1, action=OrdAction.BUY, type=OrdType.STOP,
qty=1000, stop_price=18.5)
fill_info = handler.process(order2, bar2)
self.assertFalse(handler.stop_limit_ready(order2.cl_id, order2.cl_ord_id))
self.assertEquals(None, fill_info)
fill_info = handler.process(order2, bar1)
self.assertTrue(handler.stop_limit_ready(order2.cl_id, order2.cl_ord_id))
self.assertEquals(18.5, fill_info.fill_price)
self.assertEquals(1000, fill_info.fill_qty)
# SELL
order3 = NewOrderRequest(cl_id='test', cl_ord_id=3, inst_id=1, action=OrdAction.SELL, type=OrdType.STOP,
qty=1000, stop_price=18.5)
fill_info = handler.process_w_price_qty(order3, 19, 1000)
self.assertFalse(handler.stop_limit_ready(order3.cl_id, order3.cl_ord_id))
self.assertEquals(None, fill_info)
fill_info = handler.process_w_price_qty(order3, 18, 1000)
self.assertTrue(handler.stop_limit_ready(order3.cl_id, order3.cl_ord_id))
self.assertEquals(18, fill_info.fill_price)
self.assertEquals(1000, fill_info.fill_qty)
# SELL with bar
order4 = NewOrderRequest(cl_id='test', cl_ord_id=4, inst_id=1, action=OrdAction.SELL, type=OrdType.STOP,
qty=1000, stop_price=18.5)
fill_info = handler.process(order4, bar1)
self.assertFalse(handler.stop_limit_ready(order4.cl_id, order4.cl_ord_id))
self.assertEquals(None, fill_info)
fill_info = handler.process(order4, bar2)
self.assertTrue(handler.stop_limit_ready(order4.cl_id, order4.cl_ord_id))
self.assertEquals(18.5, fill_info.fill_price)
self.assertEquals(1000, fill_info.fill_qty)
def test_tick_bar_from_trade(self):
agg = BarAggregator(data_bus=self.event_bus, clock=self.simluation_clock, inst_id=1, input=self.input,
output_bar_type=BarType.Tick, output_size=3)
agg.start()
self.assertEqual(0, len(self.event_bus.items))
self.time += 60000
t = Trade(timestamp=self.time, inst_id=1, price=20, size=200)
self.update(self.input, t)
self.assertEqual(1, agg.count())
self.assertTrue(len(self.event_bus.items) == 0)
self.time += 60000
t = Trade(timestamp=self.time, inst_id=1, price=80, size=100)
self.update(self.input, t)
self.assertEqual(2, agg.count())
self.assertTrue(len(self.event_bus.items) == 0)
self.time += 60000
t = Trade(timestamp=self.time, inst_id=1, price=10, size=200)
self.update(self.input, t)
items = self.event_bus.items
self.assertEqual(1, len(items))
self.assertEqual(0, agg.count())
self.assertEqual(
Bar(inst_id=1, begin_time=9000060000, timestamp=9000180000, type=BarType.Tick, size=3, open=20, high=80,
low=10,
close=10, vol=500, adj_close=0), items[0])
def test_subscribe_bars(self, name, datastore):
start_date = date(2011, 1, 1)
end_date = date(2011, 1, 5)
sub_key = HistDataSubscriptionKey(
inst_id=10,
provider_id=Broker.IB,
subscription_type=BarSubscriptionType(bar_type=BarType.Time,
bar_size=BarSize.D1),
from_date=start_date,
to_date=end_date)
date_val = start_date
expect_val = []
for i in range(1, 5):
persistable = Bar(timestamp=date_to_unixtimemillis(date_val),
type=BarType.Time,
size=BarSize.D1,
inst_id=10,
open=18 + i,
high=19 + i,
low=17 + i,
close=17.5 + i,
vol=100)
datastore.save_bar(persistable)
expect_val.append(persistable)
date_val = date_val + timedelta(days=1)
actual_val = datastore.load_mktdata(sub_key)
self.assertEqual(expect_val, actual_val)
def __read_csv(self, sub_keys):
dfs = []
sub_key_range = {sub_key.inst_id: (
DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for
sub_key in sub_keys}
for sub_key in sub_keys:
## TODO support different format, e.g. BAR, Quote, Trade csv files
if isinstance(sub_key.subscription_type,
BarSubscriptionType) and sub_key.subscription_type.bar_type == BarType.Time and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
df = self.read_csv(symbol, '%s/%s.csv' % (self.path, symbol.lower()))
dfs.append(df)
df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def test_save_and_load(self):
inputs = []
for x in range(0, 10):
data = sorted([random.randint(0, 100) for i in range(0, 4)])
bar = Bar(timestamp=x,
inst_id=3,
open=data[1],
high=data[3],
low=data[0],
close=data[2],
vol=random.randint(100, 1000))
inputs.append(bar)
self.db.save_bar(bar)
self.db.stop()
self.db = InMemoryDataStore()
self.db.start(self.context)
bars = self.db.load_all('bars')
bars = sorted(bars,
cmp=lambda x, y: x.timestamp - y.timestamp,
reverse=False)
self.assertEquals(10, len(bars))
for x in range(0, 10):
self.assertEquals(inputs[x], bars[x])
def test_get_bar(self):
bar = self.inst_data_mgr.get_bar(1)
self.assertIsNone(bar)
bar1 = Bar(inst_id=1, open=20, high=21, low=19, close=20.5)
self.inst_data_mgr.on_bar(bar1)
bar = self.inst_data_mgr.get_bar(1)
self.assertEqual(bar1, bar)
def test_bar(self, name, serializer):
item = Bar(open=18,
high=19,
low=17,
close=17.5,
vol=100,
inst_id=1,
timestamp=datetime.datetime.now())
SerializerTest.ser_deser(name, serializer, item)
def _start(self, app_context, **kwargs):
if self.__input is None:
self.__input = app_context.inst_data_mgr.get_series(self.input.name)
self.__input.subject.subscribe(on_next=self.on_update)
if self.__output_bar_type == BarType.Time:
current_ts = self.__clock.now()
next_ts = Bar.get_next_bar_start_time(current_ts, self.__output_size)
diff = next_ts - current_ts
Observable.timer(int(diff), self.__output_size * 1000, self.__clock.scheduler).subscribe(
on_next=self.publish)
def process_row(self, index, row):
inst = self.__ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=Clock.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def test_bar(self, name, datastore):
persistable = Bar(timestamp=clock.now(),
open=18,
high=19,
low=17,
close=17.5,
vol=100,
inst_id=999)
DataStoreTest.save_load(name, persistable, datastore,
datastore.save_bar, 'bars')
def test_limit_order_handler(self):
handler = LimitOrderHandler(self.config)
bar1 = Bar(inst_id=1, open=20, high=21, low=19, close=20.5, vol=1000)
bar2 = Bar(inst_id=1, open=16, high=18, low=15, close=17, vol=1000)
# BUY
order = NewOrderRequest(cl_id='test', cl_ord_id=1, inst_id=1, action=OrdAction.BUY, type=OrdType.LIMIT,
qty=1000, limit_price=18.5)
fill_info = handler.process_w_price_qty(order, 20, 1000)
self.assertEquals(None, fill_info)
fill_info = handler.process_w_price_qty(order, 18, 1000)
self.assertEquals(18.5, fill_info.fill_price)
self.assertEquals(1000, fill_info.fill_qty)
# BUY with bar
fill_info = handler.process(order, bar1)
self.assertEquals(None, fill_info)
fill_info = handler.process(order, bar2)
self.assertEquals(18.5, fill_info.fill_price)
self.assertEquals(1000, fill_info.fill_qty)
# SELL
order2 = NewOrderRequest(cl_id='test', cl_ord_id=2, inst_id=1, action=OrdAction.SELL, type=OrdType.LIMIT,
qty=1000,
limit_price=18.5)
fill_info = handler.process_w_price_qty(order2, 18, 1000)
self.assertEquals(None, fill_info)
fill_info = handler.process_w_price_qty(order2, 20, 1000)
self.assertEquals(18.5, fill_info.fill_price)
self.assertEquals(1000, fill_info.fill_qty)
# SELL with bar
fill_info = handler.process(order2, bar2)
self.assertEquals(None, fill_info)
fill_info = handler.process(order2, bar1)
self.assertEquals(18.5, fill_info.fill_price)
self.assertEquals(1000, fill_info.fill_qty)
def process_row(self, index, row):
logger.debug("[%s] process_row with index %s, symbol %s" %
(self.__class__.__name__, index, row['Symbol']))
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=DateUtils.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def load_bars(self, sub_key):
from_timestamp = DateUtils.date_to_unixtimemillis(sub_key.from_date)
to_timestamp = DateUtils.date_to_unixtimemillis(sub_key.to_date)
bound_stmt = self.query_bars_stmt.bind(
[sub_key.inst_id, sub_key.subscription_type.bar_type, sub_key.subscription_type.bar_size,
from_timestamp, to_timestamp])
result_list = self.session.execute(bound_stmt)
return [Bar(inst_id=r.inst_id, type=r.type, size=r.size, begin_time=r.begin_time, timestamp=r.timestamp,
open=r.open, high=r.high, low=r.low, close=r.close, vol=r.vol, adj_close=r.adj_close) for r in
result_list]
def _start(self, app_context, **kwargs):
if self.__input is None:
self.__input = app_context.inst_data_mgr.get_series(
self.input.name)
self.__input.subject.subscribe(on_next=self.on_update)
if self.__output_bar_type == BarType.Time:
current_ts = self.__clock.now()
next_ts = Bar.get_next_bar_start_time(current_ts,
self.__output_size)
diff = next_ts - current_ts
Observable.timer(
int(diff), self.__output_size * 1000,
self.__clock.scheduler).subscribe(on_next=self.publish)
def test_time_bar_from_mid(self):
agg = BarAggregator(data_bus=self.event_bus, clock=self.simluation_clock, inst_id=1, input=self.input,
input_type=BarInputType.Middle)
agg.start()
self.assertEqual(0, len(self.event_bus.items))
self.time += 59999
t = Quote(timestamp=self.time, inst_id=1, ask=30, ask_size=100, bid=18, bid_size=200)
self.update(self.input, t)
items = self.event_bus.items
self.assertEqual(1, len(items))
self.assertEqual(0, agg.count())
self.assertEqual(
Bar(inst_id=1, begin_time=9000000000, timestamp=9000059999, type=1, size=60, open=24, high=24, low=24,
close=24, vol=150, adj_close=0), items[0])
def test_bar():
config = MongoDBConfig()
store = MongoDBDataStore(config)
store.start()
for x in range(0, 10):
data = sorted([random.randint(0, 100) for i in range(0, 4)])
bar = Bar(inst_id=3,
open=data[1],
high=data[3],
low=data[0],
close=data[2],
vol=random.randint(100, 1000))
store.save(bar)
print bar
time.sleep(1)
def test1():
client = MongoClient('localhost', 27017)
db = client.market_data
bars = db.bars
# collection = db.test_collection
# for unpacked in bars.find():
# bar = Bar()
# bar.deserialize(unpacked)
# print bar
serializer = JsonSerializer()
for x in range(0, 10):
data = sorted([random.randint(0, 100) for i in range(0, 4)])
bar = Bar(inst_id=3,
open=data[1],
high=data[3],
low=data[0],
close=data[2],
vol=random.randint(100, 1000))
# print bar
packed = bar.serialize()
id = bar.id()
# if id:
# packed['_id'] = id
print packed
bars.update({'_id': id}, packed, upsert=True)
# print bar_id
# print bars.find_one()
# print bars.find_one({"inst_id": "3"})
result = bars.find_one({"_id": id})
unpacked = Bar()
unpacked.deserialize(result)
print unpacked, (unpacked == bar)
time.sleep(1)
def __load_data(self, sub_keys):
dfs = []
sub_key_range = {
sub_key.inst_id:
(DateUtils.date_to_unixtimemillis(sub_key.from_date),
DateUtils.date_to_unixtimemillis(sub_key.to_date))
for sub_key in sub_keys
}
for sub_key in sub_keys:
if not isinstance(sub_key, HistDataSubscriptionKey):
raise RuntimeError(
"only HistDataSubscriptionKey is supported!")
if isinstance(
sub_key.subscription_type, BarSubscriptionType
) and sub_key.subscription_type.bar_type == BarType.Time: #and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
# df = web.DataReader("F", self.system, sub_key.from_date, sub_key.to_date)
df = self.dict_of_df[symbol]
# df['Symbol'] = symbol
# df['BarSize'] = int(BarSize.M5)
dfs.append(df)
if len(dfs) > 0:
self.df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def test_bar_processor(self):
config = SimConfig()
processor = BarProcessor()
order = NewOrderRequest(cl_id='test',
cl_ord_id=1,
inst_id=1,
action=OrdAction.BUY,
type=OrdType.LIMIT,
qty=1000,
limit_price=18.5)
bar = Bar(open=18, high=19, low=17, close=17.5, vol=1000)
self.assertEqual(17.5, processor.get_price(order, bar, config))
self.assertEqual(1000, processor.get_qty(order, bar, config))
config2 = SimConfig(fill_on_bar_mode=SimConfig.FillMode.NEXT_OPEN)
self.assertEqual(18, processor.get_price(order, bar, config2))
self.assertEqual(1000, processor.get_qty(order, bar, config2))
def test_on_market_date_update(self):
ord_req = NewOrderRequest(cl_id='test', cl_ord_id=1, portf_id="test", broker_id="Dummy", inst_id=1,
action=OrdAction.BUY, type=OrdType.LIMIT, qty=1000, limit_price=18.5,
timestamp=0)
order1 = self.portfolio.on_new_ord_req(ord_req)
er1 = ExecutionReport(cl_id='test', cl_ord_id=1, ord_id=1, er_id=1, inst_id=1, last_qty=500, last_price=18.4,
status=OrdStatus.PARTIALLY_FILLED, timestamp=1)
self.app_context.order_mgr.on_exec_report(er1)
expected_cash = 100000 - 500 * 18.4
expected_stock_value = 500 * 18.4
expected_total_equity = expected_cash + expected_stock_value
self.assertEqual(expected_cash, self.portfolio.cash)
self.assertEqual(expected_stock_value, self.portfolio.performance_series.get_by_idx(0, 'stock_value'))
self.assertEqual(expected_total_equity, self.portfolio.performance_series.get_by_idx(0, 'total_equity'))
self.portfolio.on_trade(Trade(inst_id=1, price=20, size=1000, timestamp=2))
expected_cash = 100000 - 500 * 18.4
expected_stock_value = 500 * 20
expected_total_equity = expected_cash + expected_stock_value
self.assertEqual(expected_cash, self.portfolio.cash)
self.assertEqual(expected_stock_value, self.portfolio.performance_series.get_by_idx(1, 'stock_value'))
self.assertEqual(expected_total_equity, self.portfolio.performance_series.get_by_idx(1, 'total_equity'))
self.portfolio.on_bar(Bar(inst_id=1, close=16, adj_close=16, vol=1000, timestamp=3))
expected_cash = 100000 - 500 * 18.4
expected_stock_value = 500 * 16
expected_total_equity = expected_cash + expected_stock_value
self.assertEqual(expected_cash, self.portfolio.cash)
self.assertEqual(expected_stock_value, self.portfolio.performance_series.get_by_idx(2, 'stock_value'))
self.assertEqual(expected_total_equity, self.portfolio.performance_series.get_by_idx(2, 'total_equity'))
self.portfolio.on_quote(Quote(inst_id=1, bid=16, ask=18, timestamp=4))
expected_cash = 100000 - 500 * 18.4
expected_stock_value = 500 * 17
expected_total_equity = expected_cash + expected_stock_value
self.assertEqual(expected_cash, self.portfolio.cash)
self.assertEqual(expected_stock_value, self.portfolio.performance_series.get_by_idx(3, 'stock_value'))
self.assertEqual(expected_total_equity, self.portfolio.performance_series.get_by_idx(3, 'total_equity'))
def publish(self, *args):
if self.__count > 0:
if self.__output_bar_type == BarType.Time:
self.__timestamp = self.__end_time
bar = Bar(inst_id=self.__inst_id,
begin_time=self.__start_time,
timestamp=self.__timestamp,
open=self.__open,
high=self.__high,
low=self.__low,
close=self.__close,
vol=self.__volume,
adj_close=0,
size=self.__output_size,
type=self.__output_bar_type)
self.__data_bus.on_next(bar)
self.__reset()
def realtimeBar(self, reqId, time, open, high, low, close, volume, wap, count):
"""
TickerId reqId, long time, double open, double high, double low, double close, long volume,
double wap, int count
"""
sub_key = self.data_sub_reg.get_subscription_key(reqId)
record = self.data_sub_reg.get_data_record(reqId)
if record:
record.open = open
record.high = high
record.low = low
record.close = close
record.vol = volume
timestamp = self.model_factory.convert_ib_time(time)
self.data_event_bus.on_next(
Bar(inst_id=record.inst_id, timestamp=timestamp, open=open, high=high, low=low, close=close,
vol=volume, size=sub_key.subscription_type.bar_size))
def test_time_bar_from_trade(self):
agg = BarAggregator(data_bus=self.event_bus, clock=self.simluation_clock, inst_id=1, input=self.input)
agg.start()
self.assertEqual(0, len(self.event_bus.items))
self.time += 10000
t = Trade(timestamp=self.time, inst_id=1, price=20, size=200)
self.update(self.input, t)
self.assertEqual(1, agg.count())
self.assertTrue(len(self.event_bus.items) == 0)
# expect get a aggregated bar at 9000059999
self.time += 49999
t = Trade(timestamp=self.time, inst_id=1, price=10, size=200)
self.update(self.input, t)
items = self.event_bus.items
self.assertEqual(1, len(items))
self.assertEqual(0, agg.count())
self.assertEqual(
Bar(inst_id=1, begin_time=9000000000, timestamp=9000059999, type=1, size=60, open=20, high=20, low=10,
close=10, vol=400, adj_close=0), items[0])
def test_time_bar_from_bid(self):
agg = BarAggregator(data_bus=self.event_bus, clock=self.simluation_clock, inst_id=1, input=self.input,
input_type=BarInputType.Bid)
agg.start()
self.assertEqual(0, len(self.event_bus.items))
self.time += 10000
t = Quote(timestamp=self.time, inst_id=1, bid=30, bid_size=100)
self.update(self.input, t)
self.assertEqual(1, agg.count())
self.assertTrue(len(self.event_bus.items) == 0)
self.time += 10000
t = Quote(timestamp=self.time, inst_id=1, bid=10, bid_size=200)
self.update(self.input, t)
self.assertEqual(2, agg.count())
self.assertTrue(len(self.event_bus.items) == 0)
self.time += 10000
t = Quote(timestamp=self.time, inst_id=1, bid=70, bid_size=300)
self.update(self.input, t)
self.assertEqual(3, agg.count())
self.assertTrue(len(self.event_bus.items) == 0)
self.time += 29998
t = Quote(timestamp=self.time, inst_id=1, bid=50, bid_size=400)
self.update(self.input, t)
self.assertEqual(4, agg.count())
self.assertTrue(len(self.event_bus.items) == 0)
self.time += 3
self.simluation_clock.update_time(self.time)
items = self.event_bus.items
self.assertEqual(1, len(items))
self.assertEqual(0, agg.count())
self.assertEqual(
Bar(inst_id=1, begin_time=9000000000, timestamp=9000059999, type=1, size=60, open=30, high=70, low=10,
close=50, vol=1000, adj_close=0), items[0])
def historicalData(self, reqId, date, open, high,
low, close, volume,
barCount, WAP, hasGaps):
"""
TickerId reqId, IBString const & date, double open, double high, double low, double close,
int volume, int barCount, double WAP, int hasGaps)
"""
if barCount < 0:
return
sub_key = self.data_sub_reg.get_subscription_key(reqId)
record = self.data_sub_reg.get_data_record(reqId)
if record:
record.open = open
record.high = high
record.low = low
record.close = close
record.vol = volume
timestamp = self.model_factory.convert_ib_date(date)
self.data_event_bus.on_next(
Bar(inst_id=record.inst_id, timestamp=timestamp, open=open, high=high, low=low,
close=close, vol=volume, size=sub_key.bar_size))
def load_all(self, clazz):
result_list = self.session.execute("select * from %s " % clazz)
if clazz == 'sequences':
return {result.id: result.seq for result in result_list}
else:
results = []
for r in result_list:
if clazz == 'bars':
obj = Bar(inst_id=r.inst_id, type=r.type, size=r.size, begin_time=r.begin_time,
timestamp=r.timestamp,
open=r.open, high=r.high, low=r.low, close=r.close, vol=r.vol, adj_close=r.adj_close)
elif clazz == 'trades':
obj = Trade(inst_id=r.inst_id, timestamp=r.timestamp, price=r.price, size=r.size)
elif clazz == 'quotes':
obj = Quote(inst_id=r.inst_id, timestamp=r.timestamp, bid=r.bid, ask=r.ask, bid_size=r.bid_size,
ask_size=r.ask_size)
elif clazz == 'market_depths':
obj = MarketDepth(inst_id=r.inst_id, provider_id=r.provider_id, timestamp=r.timestamp,
position=r.position, operation=r.operation, side=r.side, price=r.price,
size=r.size)
elif clazz == 'instruments':
obj = Instrument(inst_id=r.inst_id, name=r.name, type=r.type, symbol=r.symbol, exch_id=r.exch_id,
ccy_id=r.ccy_id, alt_symbol=r.alt_symbol, alt_exch_id=r.alt_exch_id,
sector=r.sector,
industry=r.industry, und_inst_id=r.und_inst_id, expiry_date=r.expiry_date,
factor=r.factor,
strike=r.strike, put_call=r.put_call, margin=r.margin)
elif clazz == 'currencies':
obj = Currency(ccy_id=r.ccy_id, name=r.name)
elif clazz == 'exchanges':
obj = Exchange(exch_id=r.exch_id, name=r.name)
else:
obj = self.serializer.deserialize(r.data)
results.append(obj)
return results
def test1():
client = MongoClient('localhost', 27017)
db = client.market_data
bars = db.bars
# collection = db.test_collection
# for unpacked in bars.find():
# bar = Bar()
# bar.deserialize(unpacked)
# print bar
serializer = JsonSerializer()
for x in range(0, 10):
data = sorted([random.randint(0, 100) for i in range(0, 4)])
bar = Bar(inst_id=3, open=data[1], high=data[3], low=data[0], close=data[2], vol=random.randint(100, 1000))
# print bar
packed = bar.serialize()
id = bar.id()
# if id:
# packed['_id'] = id
print packed
bars.update({'_id': id}, packed, upsert=True)
# print bar_id
# print bars.find_one()
# print bars.find_one({"inst_id": "3"})
result = bars.find_one({"_id": id})
unpacked = Bar()
unpacked.deserialize(result)
print unpacked, (unpacked == bar)
time.sleep(1)
class MyNoopGreenlet(Greenlet):
def __init__(self, seconds):
Greenlet.__init__(self)
self.seconds = seconds
def _run(self):
gevent.sleep(self.seconds)
def __str__(self):
return 'MyNoopGreenlet(%s)' % self.seconds
current_ts = DateUtils.datetime_to_unixtimemillis(starttime)
next_ts = Bar.get_next_bar_start_time(current_ts, BarSize.S5)
diff = next_ts - current_ts
# Observable.timer(int(diff), BarSize.S5 * 1000, scheduler2).subscribe(action)
# scheduler1.advance_to(starttime)
# scheduler2.schedule_absolute(datetime.utcnow() + timedelta(seconds=3), action, scheduler2.now)
# print "1", scheduler1.now()
# scheduler1.advance_to(starttime + timedelta(seconds=1))
# print "2", scheduler1.now()
# scheduler1.advance_to(starttime + timedelta(seconds=2))
# print "3", scheduler1.now()
# scheduler1.advance_to(starttime + timedelta(seconds=3))
# print "4", scheduler1.now()
# scheduler1.advance_by(2000)
# print "5", scheduler1.now()
def test_multi_subscriptions(self, name, datastore):
start_date = date(2011, 1, 1)
end_date = date(2011, 1, 5)
sub_key1 = HistDataSubscriptionKey(
inst_id=99,
provider_id=Broker.IB,
subscription_type=BarSubscriptionType(bar_type=BarType.Time,
bar_size=BarSize.D1),
from_date=start_date,
to_date=end_date)
sub_key2 = HistDataSubscriptionKey(
inst_id=99,
provider_id=Broker.IB,
subscription_type=QuoteSubscriptionType(),
from_date=start_date,
to_date=end_date)
sub_key3 = HistDataSubscriptionKey(
inst_id=99,
provider_id=Broker.IB,
subscription_type=TradeSubscriptionType(),
from_date=start_date,
to_date=end_date)
expect_val = []
# out of range
persistable = Bar(timestamp=date_to_unixtimemillis(date(2010, 12, 31)),
type=BarType.Time,
size=BarSize.D1,
inst_id=99,
open=18,
high=19,
low=17,
close=17.5,
vol=100)
datastore.save_bar(persistable)
persistable = Bar(timestamp=date_to_unixtimemillis(date(2011, 1, 1)),
type=BarType.Time,
size=BarSize.D1,
inst_id=99,
open=28,
high=29,
low=27,
close=27.5,
vol=100)
datastore.save_bar(persistable)
expect_val.append(persistable)
persistable = Trade(timestamp=date_to_unixtimemillis(date(2011, 1, 2)),
price=20,
size=200,
inst_id=99)
datastore.save_trade(persistable)
expect_val.append(persistable)
persistable = Trade(timestamp=date_to_unixtimemillis(date(2011, 1, 3)),
price=30,
size=200,
inst_id=99)
datastore.save_trade(persistable)
expect_val.append(persistable)
# not same instrument
persistable = Quote(timestamp=date_to_unixtimemillis(date(2011, 1, 3)),
bid=18,
ask=19,
bid_size=200,
ask_size=500,
inst_id=11)
datastore.save_quote(persistable)
persistable = Quote(timestamp=date_to_unixtimemillis(date(2011, 1, 4)),
bid=18,
ask=19,
bid_size=200,
ask_size=500,
inst_id=99)
datastore.save_quote(persistable)
expect_val.append(persistable)
# out of range
persistable = Quote(timestamp=date_to_unixtimemillis(date(2011, 1, 5)),
bid=28,
ask=29,
bid_size=200,
ask_size=500,
inst_id=99)
datastore.save_quote(persistable)
actual_val = datastore.load_mktdata(sub_key1, sub_key2, sub_key3)
self.assertEqual(expect_val, actual_val)
