def test_timestamp_conversion(self): dt = datetime.datetime(year=2000, month=1, day=1, hour=7, minute=30, second=30) ts = DateUtils.datetime_to_unixtimemillis(dt) self.assertEqual(946683030000, ts) dt2 = DateUtils.unixtimemillis_to_datetime(ts) self.assertEquals(dt, dt2) dt3 = datetime.datetime.fromtimestamp(0) ts2 = DateUtils.datetime_to_unixtimemillis(dt3) dt4 = DateUtils.unixtimemillis_to_datetime(ts2) self.assertEquals(0, ts2) self.assertEquals(dt3, dt4)
def __read_csv(self, sub_keys): dfs = [] sub_key_range = {sub_key.inst_id: ( DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for sub_key in sub_keys} for sub_key in sub_keys: ## TODO support different format, e.g. BAR, Quote, Trade csv files if isinstance(sub_key.subscription_type, BarSubscriptionType) and sub_key.subscription_type.bar_type == BarType.Time and sub_key.subscription_type.bar_size == BarSize.D1: inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id) symbol = inst.get_symbol(self.id()) df = self.read_csv(symbol, '%s/%s.csv' % (self.path, symbol.lower())) dfs.append(df) df = pd.concat(dfs).sort_index(0, ascending=True) for index, row in df.iterrows(): inst = self.ref_data_mgr.get_inst(symbol=row['Symbol']) range = sub_key_range[inst.inst_id] timestamp = DateUtils.datetime_to_unixtimemillis(index) if timestamp >= range[0] and timestamp < range[1]: self.data_event_bus.on_next( Bar(inst_id=inst.inst_id, timestamp=timestamp, open=row['Open'], high=row['High'], low=row['Low'], close=row['Close'], vol=row['Volume'], adj_close=row['Adj Close'], size=row['BarSize']))
def test_real_time_clock_now(self): ts = DateUtils.datetime_to_unixtimemillis(datetime.datetime.now()) ts2 = self.realtime_clock.now() self.assertTrue(abs(ts - ts2) <= 10) time.sleep(2) ts3 = self.realtime_clock.now() self.assertAlmostEqual(ts3 - ts2, 2000, -2)
def get_current_bar_start_time(timestamp, bar_size): if bar_size < BarSize.D1: return int(timestamp / (bar_size * 1000)) * bar_size * 1000 else: dt = datetime.datetime.fromtimestamp(timestamp / 1000) return DateUtils.datetime_to_unixtimemillis( datetime.datetime(year=dt.year, month=dt.month, day=dt.day))
def test_real_time_clock_now(self): ts = DateUtils.datetime_to_unixtimemillis(datetime.datetime.now()) ts2 = self.realtime_clock.now() self.assertTrue(abs(ts - ts2) <= 10) time.sleep(2) ts3 = self.realtime_clock.now() self.assertAlmostEqual(ts3 - ts2, 2000, -2)
def test_timestamp_conversion(self): dt = datetime.datetime(year=2000, month=1, day=1, hour=7, minute=30, second=30) ts = DateUtils.datetime_to_unixtimemillis(dt) self.assertEqual(946683030000, ts) dt2 = DateUtils.unixtimemillis_to_datetime(ts) self.assertEquals(dt, dt2) dt3 = datetime.datetime.fromtimestamp(0) ts2 = DateUtils.datetime_to_unixtimemillis(dt3) dt4 = DateUtils.unixtimemillis_to_datetime(ts2) self.assertEquals(0, ts2) self.assertEquals(dt3, dt4)
def process_row(self, index, row): inst = self.ref_data_mgr.get_inst(symbol=row['Symbol']) return Bar(inst_id=inst.inst_id, timestamp=DateUtils.datetime_to_unixtimemillis(index), open=row['Open'], high=row['High'], low=row['Low'], close=row['Close'], vol=row['Volume'], size=row['BarSize'])
def process_row(self, index, row): inst = self.ref_data_mgr.get_inst(symbol=row['Symbol']) return Bar(inst_id=inst.inst_id, timestamp=DateUtils.datetime_to_unixtimemillis(index), open=row['Open'], high=row['High'], low=row['Low'], close=row['Close'], vol=row['Volume'], size=row['BarSize'])
def process_row(self, index, row): logger.debug("[%s] process_row with index %s, symbol %s" % (self.__class__.__name__, index, row['Symbol'])) inst = self.ref_data_mgr.get_inst(symbol=row['Symbol']) return Bar(inst_id=inst.inst_id, timestamp=DateUtils.datetime_to_unixtimemillis(index), open=row['Open'], high=row['High'], low=row['Low'], close=row['Close'], vol=row['Volume'], size=row['BarSize'])
def expected(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None): year = year if year >= 0 else BarTest.current_dt.year month = month if month >= 0 else BarTest.current_dt.month day = day if day >= 0 else BarTest.current_dt.day hour = hour if hour >= 0 else BarTest.current_dt.hour minute = minute if minute >= 0 else BarTest.current_dt.minute second = second if second >= 0 else BarTest.current_dt.second microsecond = microsecond if microsecond >= 0 else BarTest.current_dt.microsecond return DateUtils.datetime_to_unixtimemillis( datetime.datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second, microsecond=microsecond))
def process_row(self, index, row): logger.debug("[%s] process_row with index %s, symbol %s" % (self.__class__.__name__, index, row['Symbol'])) inst = self.ref_data_mgr.get_inst(symbol=row['Symbol']) return Bar(inst_id=inst.inst_id, timestamp=DateUtils.datetime_to_unixtimemillis(index), open=row['Open'], high=row['High'], low=row['Low'], close=row['Close'], vol=row['Volume'], size=row['BarSize'])
def __load_data(self, sub_keys): dfs = [] sub_key_range = { sub_key.inst_id: (DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for sub_key in sub_keys } for sub_key in sub_keys: if not isinstance(sub_key, HistDataSubscriptionKey): raise RuntimeError( "only HistDataSubscriptionKey is supported!") if isinstance( sub_key.subscription_type, BarSubscriptionType ) and sub_key.subscription_type.bar_type == BarType.Time: #and sub_key.subscription_type.bar_size == BarSize.D1: inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id) symbol = inst.get_symbol(self.id()) # df = web.DataReader("F", self.system, sub_key.from_date, sub_key.to_date) df = self.dict_of_df[symbol] # df['Symbol'] = symbol # df['BarSize'] = int(BarSize.M5) dfs.append(df) if len(dfs) > 0: self.df = pd.concat(dfs).sort_index(0, ascending=True) for index, row in df.iterrows(): inst = self.ref_data_mgr.get_inst(symbol=row['Symbol']) range = sub_key_range[inst.inst_id] timestamp = DateUtils.datetime_to_unixtimemillis(index) if timestamp >= range[0] and timestamp < range[1]: self.data_event_bus.on_next( Bar(inst_id=inst.inst_id, timestamp=timestamp, open=row['Open'], high=row['High'], low=row['Low'], close=row['Close'], vol=row['Volume'], adj_close=row['Adj Close'], size=row['BarSize']))
def __load_data(self, sub_keys): dfs = [] sub_key_range = {sub_key.inst_id: ( DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for sub_key in sub_keys} for sub_key in sub_keys: if not isinstance(sub_key, HistDataSubscriptionKey): raise RuntimeError("only HistDataSubscriptionKey is supported!") if isinstance(sub_key.subscription_type, BarSubscriptionType) and sub_key.subscription_type.bar_type == BarType.Time: #and sub_key.subscription_type.bar_size == BarSize.D1: inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id) symbol = inst.get_symbol(self.id()) # df = web.DataReader("F", self.system, sub_key.from_date, sub_key.to_date) df = self.dict_of_df[symbol] # df['Symbol'] = symbol # df['BarSize'] = int(BarSize.M5) dfs.append(df) if len(dfs) > 0: self.df = pd.concat(dfs).sort_index(0, ascending=True) for index, row in df.iterrows(): inst = self.ref_data_mgr.get_inst(symbol=row['Symbol']) range = sub_key_range[inst.inst_id] timestamp = DateUtils.datetime_to_unixtimemillis(index) if timestamp >= range[0] and timestamp < range[1]: self.data_event_bus.on_next( Bar(inst_id=inst.inst_id, timestamp=timestamp, open=row['Open'], high=row['High'], low=row['Low'], close=row['Close'], vol=row['Volume'], adj_close=row['Adj Close'], size=row['BarSize']))
def expected(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None): year = year if year >= 0 else BarTest.current_dt.year month = month if month >= 0 else BarTest.current_dt.month day = day if day >= 0 else BarTest.current_dt.day hour = hour if hour >= 0 else BarTest.current_dt.hour minute = minute if minute >= 0 else BarTest.current_dt.minute second = second if second >= 0 else BarTest.current_dt.second microsecond = microsecond if microsecond >= 0 else BarTest.current_dt.microsecond return DateUtils.datetime_to_unixtimemillis( datetime.datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second, microsecond=microsecond))
def convert_ib_datetime(self, ib_datetime_str): return DateUtils.datetime_to_unixtimemillis(datetime.strptime(ib_datetime_str, self.IB_DATETIME_FORMAT2))
def convert_ib_datetime(self, ib_datetime_str): return DateUtils.datetime_to_unixtimemillis( datetime.strptime(ib_datetime_str, self.IB_DATETIME_FORMAT2))
from gevent.greenlet import Greenlet class MyNoopGreenlet(Greenlet): def __init__(self, seconds): Greenlet.__init__(self) self.seconds = seconds def _run(self): gevent.sleep(self.seconds) def __str__(self): return 'MyNoopGreenlet(%s)' % self.seconds current_ts = DateUtils.datetime_to_unixtimemillis(starttime) next_ts = Bar.get_next_bar_start_time(current_ts, BarSize.S5) diff = next_ts - current_ts # Observable.timer(int(diff), BarSize.S5 * 1000, scheduler2).subscribe(action) # scheduler1.advance_to(starttime) # scheduler2.schedule_absolute(datetime.utcnow() + timedelta(seconds=3), action, scheduler2.now) # print "1", scheduler1.now() # scheduler1.advance_to(starttime + timedelta(seconds=1)) # print "2", scheduler1.now() # scheduler1.advance_to(starttime + timedelta(seconds=2)) # print "3", scheduler1.now() # scheduler1.advance_to(starttime + timedelta(seconds=3)) # print "4", scheduler1.now() # scheduler1.advance_by(2000) # print "5", scheduler1.now()
def convert_ib_time(self, ib_time): return DateUtils.datetime_to_unixtimemillis( datetime.fromtimestamp(ib_time))
from gevent.greenlet import Greenlet class MyNoopGreenlet(Greenlet): def __init__(self, seconds): Greenlet.__init__(self) self.seconds = seconds def _run(self): gevent.sleep(self.seconds) def __str__(self): return 'MyNoopGreenlet(%s)' % self.seconds current_ts = DateUtils.datetime_to_unixtimemillis(starttime) next_ts = Bar.get_next_bar_start_time(current_ts, BarSize.S5) diff = next_ts - current_ts # Observable.timer(int(diff), BarSize.S5 * 1000, scheduler2).subscribe(action) # scheduler1.advance_to(starttime) # scheduler2.schedule_absolute(datetime.utcnow() + timedelta(seconds=3), action, scheduler2.now) # print "1", scheduler1.now() # scheduler1.advance_to(starttime + timedelta(seconds=1)) # print "2", scheduler1.now() # scheduler1.advance_to(starttime + timedelta(seconds=2)) # print "3", scheduler1.now() # scheduler1.advance_to(starttime + timedelta(seconds=3)) # print "4", scheduler1.now() # scheduler1.advance_by(2000) # print "5", scheduler1.now()
def convert_ib_time(self, ib_time): return DateUtils.datetime_to_unixtimemillis(datetime.fromtimestamp(ib_time))
class BarTest(TestCase): current_dt = datetime.datetime(year=2016, month=8, day=1, hour=6, minute=3, second=4) current_ts = DateUtils.datetime_to_unixtimemillis(current_dt) def ts(self, func, size): return func(BarTest.current_ts, size) def expected(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None): year = year if year >= 0 else BarTest.current_dt.year month = month if month >= 0 else BarTest.current_dt.month day = day if day >= 0 else BarTest.current_dt.day hour = hour if hour >= 0 else BarTest.current_dt.hour minute = minute if minute >= 0 else BarTest.current_dt.minute second = second if second >= 0 else BarTest.current_dt.second microsecond = microsecond if microsecond >= 0 else BarTest.current_dt.microsecond return DateUtils.datetime_to_unixtimemillis( datetime.datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second, microsecond=microsecond)) def test_current_bar_start_time(self): func = Bar.get_current_bar_start_time self.assertEqual(self.expected(second=4), self.ts(func, BarSize.S1)) self.assertEqual(self.expected(second=0, microsecond=0), self.ts(func, BarSize.S5)) self.assertEqual(self.expected(second=0, microsecond=0), self.ts(func, BarSize.S15)) self.assertEqual(self.expected(second=0, microsecond=0), self.ts(func, BarSize.S30)) self.assertEqual(self.expected(second=0, microsecond=0), self.ts(func, BarSize.M1)) self.assertEqual(self.expected(minute=0, second=0, microsecond=0), self.ts(func, BarSize.M5)) self.assertEqual(self.expected(minute=0, second=0, microsecond=0), self.ts(func, BarSize.M15)) self.assertEqual(self.expected(minute=0, second=0, microsecond=0), self.ts(func, BarSize.M30)) self.assertEqual(self.expected(minute=0, second=0, microsecond=0), self.ts(func, BarSize.H1)) self.assertEqual( self.expected(hour=0, minute=0, second=0, microsecond=0), self.ts(func, BarSize.D1)) def test_current_bar_end_time(self): func = Bar.get_current_bar_end_time self.assertEqual(self.expected(second=4, microsecond=999999), self.ts(func, BarSize.S1)) self.assertEqual(self.expected(second=4, microsecond=999999), self.ts(func, BarSize.S5)) self.assertEqual(self.expected(second=14, microsecond=999999), self.ts(func, BarSize.S15)) self.assertEqual(self.expected(second=29, microsecond=999999), self.ts(func, BarSize.S30)) self.assertEqual(self.expected(second=59, microsecond=999999), self.ts(func, BarSize.M1)) self.assertEqual( self.expected(minute=4, second=59, microsecond=999999), self.ts(func, BarSize.M5)) self.assertEqual( self.expected(minute=14, second=59, microsecond=999999), self.ts(func, BarSize.M15)) self.assertEqual( self.expected(minute=29, second=59, microsecond=999999), self.ts(func, BarSize.M30)) self.assertEqual( self.expected(minute=59, second=59, microsecond=999999), self.ts(func, BarSize.H1)) self.assertEqual( self.expected(hour=23, minute=59, second=59, microsecond=999999), self.ts(func, BarSize.D1)) def test_next_bar_start_time(self): func = Bar.get_next_bar_start_time self.assertEqual(self.expected(second=5, microsecond=0), self.ts(func, BarSize.S1)) self.assertEqual(self.expected(second=5, microsecond=0), self.ts(func, BarSize.S5)) self.assertEqual(self.expected(second=15, microsecond=0), self.ts(func, BarSize.S15)) self.assertEqual(self.expected(second=30, microsecond=0), self.ts(func, BarSize.S30)) self.assertEqual(self.expected(minute=4, second=0, microsecond=0), self.ts(func, BarSize.M1)) self.assertEqual(self.expected(minute=5, second=0, microsecond=0), self.ts(func, BarSize.M5)) self.assertEqual(self.expected(minute=15, second=0, microsecond=0), self.ts(func, BarSize.M15)) self.assertEqual(self.expected(minute=30, second=0, microsecond=0), self.ts(func, BarSize.M30)) self.assertEqual( self.expected(hour=7, minute=0, second=0, microsecond=0), self.ts(func, BarSize.H1)) self.assertEqual( self.expected(day=2, hour=0, minute=0, second=0, microsecond=0), self.ts(func, BarSize.D1))
def get_current_bar_start_time(timestamp, bar_size): if bar_size < BarSize.D1: return int(timestamp / (bar_size * 1000)) * bar_size * 1000 else: dt = datetime.datetime.fromtimestamp(timestamp / 1000) return DateUtils.datetime_to_unixtimemillis(datetime.datetime(year=dt.year, month=dt.month, day=dt.day))