def test_timestamp_conversion(self):
dt = datetime.datetime(year=2000, month=1, day=1, hour=7, minute=30, second=30)
ts = DateUtils.datetime_to_unixtimemillis(dt)
self.assertEqual(946683030000, ts)
dt2 = DateUtils.unixtimemillis_to_datetime(ts)
self.assertEquals(dt, dt2)
dt3 = datetime.datetime.fromtimestamp(0)
ts2 = DateUtils.datetime_to_unixtimemillis(dt3)
dt4 = DateUtils.unixtimemillis_to_datetime(ts2)
self.assertEquals(0, ts2)
self.assertEquals(dt3, dt4)
def __read_csv(self, sub_keys):
dfs = []
sub_key_range = {sub_key.inst_id: (
DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for
sub_key in sub_keys}
for sub_key in sub_keys:
## TODO support different format, e.g. BAR, Quote, Trade csv files
if isinstance(sub_key.subscription_type,
BarSubscriptionType) and sub_key.subscription_type.bar_type == BarType.Time and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
df = self.read_csv(symbol, '%s/%s.csv' % (self.path, symbol.lower()))
dfs.append(df)
df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def test_real_time_clock_now(self):
ts = DateUtils.datetime_to_unixtimemillis(datetime.datetime.now())
ts2 = self.realtime_clock.now()
self.assertTrue(abs(ts - ts2) <= 10)
time.sleep(2)
ts3 = self.realtime_clock.now()
self.assertAlmostEqual(ts3 - ts2, 2000, -2)
def get_current_bar_start_time(timestamp, bar_size):
if bar_size < BarSize.D1:
return int(timestamp / (bar_size * 1000)) * bar_size * 1000
else:
dt = datetime.datetime.fromtimestamp(timestamp / 1000)
return DateUtils.datetime_to_unixtimemillis(
datetime.datetime(year=dt.year, month=dt.month, day=dt.day))
def test_real_time_clock_now(self):
ts = DateUtils.datetime_to_unixtimemillis(datetime.datetime.now())
ts2 = self.realtime_clock.now()
self.assertTrue(abs(ts - ts2) <= 10)
time.sleep(2)
ts3 = self.realtime_clock.now()
self.assertAlmostEqual(ts3 - ts2, 2000, -2)
def test_timestamp_conversion(self):
dt = datetime.datetime(year=2000,
month=1,
day=1,
hour=7,
minute=30,
second=30)
ts = DateUtils.datetime_to_unixtimemillis(dt)
self.assertEqual(946683030000, ts)
dt2 = DateUtils.unixtimemillis_to_datetime(ts)
self.assertEquals(dt, dt2)
dt3 = datetime.datetime.fromtimestamp(0)
ts2 = DateUtils.datetime_to_unixtimemillis(dt3)
dt4 = DateUtils.unixtimemillis_to_datetime(ts2)
self.assertEquals(0, ts2)
self.assertEquals(dt3, dt4)
def process_row(self, index, row):
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=DateUtils.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def process_row(self, index, row):
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=DateUtils.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def process_row(self, index, row):
logger.debug("[%s] process_row with index %s, symbol %s" % (self.__class__.__name__, index, row['Symbol']))
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=DateUtils.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def expected(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None):
year = year if year >= 0 else BarTest.current_dt.year
month = month if month >= 0 else BarTest.current_dt.month
day = day if day >= 0 else BarTest.current_dt.day
hour = hour if hour >= 0 else BarTest.current_dt.hour
minute = minute if minute >= 0 else BarTest.current_dt.minute
second = second if second >= 0 else BarTest.current_dt.second
microsecond = microsecond if microsecond >= 0 else BarTest.current_dt.microsecond
return DateUtils.datetime_to_unixtimemillis(
datetime.datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second,
microsecond=microsecond))
def process_row(self, index, row):
logger.debug("[%s] process_row with index %s, symbol %s" %
(self.__class__.__name__, index, row['Symbol']))
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
return Bar(inst_id=inst.inst_id,
timestamp=DateUtils.datetime_to_unixtimemillis(index),
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
size=row['BarSize'])
def __load_data(self, sub_keys):
dfs = []
sub_key_range = {
sub_key.inst_id:
(DateUtils.date_to_unixtimemillis(sub_key.from_date),
DateUtils.date_to_unixtimemillis(sub_key.to_date))
for sub_key in sub_keys
}
for sub_key in sub_keys:
if not isinstance(sub_key, HistDataSubscriptionKey):
raise RuntimeError(
"only HistDataSubscriptionKey is supported!")
if isinstance(
sub_key.subscription_type, BarSubscriptionType
) and sub_key.subscription_type.bar_type == BarType.Time: #and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
# df = web.DataReader("F", self.system, sub_key.from_date, sub_key.to_date)
df = self.dict_of_df[symbol]
# df['Symbol'] = symbol
# df['BarSize'] = int(BarSize.M5)
dfs.append(df)
if len(dfs) > 0:
self.df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def __load_data(self, sub_keys):
dfs = []
sub_key_range = {sub_key.inst_id: (
DateUtils.date_to_unixtimemillis(sub_key.from_date), DateUtils.date_to_unixtimemillis(sub_key.to_date)) for
sub_key in sub_keys}
for sub_key in sub_keys:
if not isinstance(sub_key, HistDataSubscriptionKey):
raise RuntimeError("only HistDataSubscriptionKey is supported!")
if isinstance(sub_key.subscription_type,
BarSubscriptionType) and sub_key.subscription_type.bar_type == BarType.Time: #and sub_key.subscription_type.bar_size == BarSize.D1:
inst = self.ref_data_mgr.get_inst(inst_id=sub_key.inst_id)
symbol = inst.get_symbol(self.id())
# df = web.DataReader("F", self.system, sub_key.from_date, sub_key.to_date)
df = self.dict_of_df[symbol]
# df['Symbol'] = symbol
# df['BarSize'] = int(BarSize.M5)
dfs.append(df)
if len(dfs) > 0:
self.df = pd.concat(dfs).sort_index(0, ascending=True)
for index, row in df.iterrows():
inst = self.ref_data_mgr.get_inst(symbol=row['Symbol'])
range = sub_key_range[inst.inst_id]
timestamp = DateUtils.datetime_to_unixtimemillis(index)
if timestamp >= range[0] and timestamp < range[1]:
self.data_event_bus.on_next(
Bar(inst_id=inst.inst_id,
timestamp=timestamp,
open=row['Open'],
high=row['High'],
low=row['Low'],
close=row['Close'],
vol=row['Volume'],
adj_close=row['Adj Close'],
size=row['BarSize']))
def expected(self,
year=None,
month=None,
day=None,
hour=None,
minute=None,
second=None,
microsecond=None):
year = year if year >= 0 else BarTest.current_dt.year
month = month if month >= 0 else BarTest.current_dt.month
day = day if day >= 0 else BarTest.current_dt.day
hour = hour if hour >= 0 else BarTest.current_dt.hour
minute = minute if minute >= 0 else BarTest.current_dt.minute
second = second if second >= 0 else BarTest.current_dt.second
microsecond = microsecond if microsecond >= 0 else BarTest.current_dt.microsecond
return DateUtils.datetime_to_unixtimemillis(
datetime.datetime(year=year,
month=month,
day=day,
hour=hour,
minute=minute,
second=second,
microsecond=microsecond))
def convert_ib_datetime(self, ib_datetime_str):
return DateUtils.datetime_to_unixtimemillis(datetime.strptime(ib_datetime_str, self.IB_DATETIME_FORMAT2))
def convert_ib_datetime(self, ib_datetime_str):
return DateUtils.datetime_to_unixtimemillis(
datetime.strptime(ib_datetime_str, self.IB_DATETIME_FORMAT2))
from gevent.greenlet import Greenlet
class MyNoopGreenlet(Greenlet):
def __init__(self, seconds):
Greenlet.__init__(self)
self.seconds = seconds
def _run(self):
gevent.sleep(self.seconds)
def __str__(self):
return 'MyNoopGreenlet(%s)' % self.seconds
current_ts = DateUtils.datetime_to_unixtimemillis(starttime)
next_ts = Bar.get_next_bar_start_time(current_ts, BarSize.S5)
diff = next_ts - current_ts
# Observable.timer(int(diff), BarSize.S5 * 1000, scheduler2).subscribe(action)
# scheduler1.advance_to(starttime)
# scheduler2.schedule_absolute(datetime.utcnow() + timedelta(seconds=3), action, scheduler2.now)
# print "1", scheduler1.now()
# scheduler1.advance_to(starttime + timedelta(seconds=1))
# print "2", scheduler1.now()
# scheduler1.advance_to(starttime + timedelta(seconds=2))
# print "3", scheduler1.now()
# scheduler1.advance_to(starttime + timedelta(seconds=3))
# print "4", scheduler1.now()
# scheduler1.advance_by(2000)
# print "5", scheduler1.now()
def convert_ib_time(self, ib_time):
return DateUtils.datetime_to_unixtimemillis(
datetime.fromtimestamp(ib_time))
from gevent.greenlet import Greenlet
class MyNoopGreenlet(Greenlet):
def __init__(self, seconds):
Greenlet.__init__(self)
self.seconds = seconds
def _run(self):
gevent.sleep(self.seconds)
def __str__(self):
return 'MyNoopGreenlet(%s)' % self.seconds
current_ts = DateUtils.datetime_to_unixtimemillis(starttime)
next_ts = Bar.get_next_bar_start_time(current_ts, BarSize.S5)
diff = next_ts - current_ts
# Observable.timer(int(diff), BarSize.S5 * 1000, scheduler2).subscribe(action)
# scheduler1.advance_to(starttime)
# scheduler2.schedule_absolute(datetime.utcnow() + timedelta(seconds=3), action, scheduler2.now)
# print "1", scheduler1.now()
# scheduler1.advance_to(starttime + timedelta(seconds=1))
# print "2", scheduler1.now()
# scheduler1.advance_to(starttime + timedelta(seconds=2))
# print "3", scheduler1.now()
# scheduler1.advance_to(starttime + timedelta(seconds=3))
# print "4", scheduler1.now()
# scheduler1.advance_by(2000)
# print "5", scheduler1.now()
def convert_ib_time(self, ib_time):
return DateUtils.datetime_to_unixtimemillis(datetime.fromtimestamp(ib_time))
class BarTest(TestCase):
current_dt = datetime.datetime(year=2016,
month=8,
day=1,
hour=6,
minute=3,
second=4)
current_ts = DateUtils.datetime_to_unixtimemillis(current_dt)
def ts(self, func, size):
return func(BarTest.current_ts, size)
def expected(self,
year=None,
month=None,
day=None,
hour=None,
minute=None,
second=None,
microsecond=None):
year = year if year >= 0 else BarTest.current_dt.year
month = month if month >= 0 else BarTest.current_dt.month
day = day if day >= 0 else BarTest.current_dt.day
hour = hour if hour >= 0 else BarTest.current_dt.hour
minute = minute if minute >= 0 else BarTest.current_dt.minute
second = second if second >= 0 else BarTest.current_dt.second
microsecond = microsecond if microsecond >= 0 else BarTest.current_dt.microsecond
return DateUtils.datetime_to_unixtimemillis(
datetime.datetime(year=year,
month=month,
day=day,
hour=hour,
minute=minute,
second=second,
microsecond=microsecond))
def test_current_bar_start_time(self):
func = Bar.get_current_bar_start_time
self.assertEqual(self.expected(second=4), self.ts(func, BarSize.S1))
self.assertEqual(self.expected(second=0, microsecond=0),
self.ts(func, BarSize.S5))
self.assertEqual(self.expected(second=0, microsecond=0),
self.ts(func, BarSize.S15))
self.assertEqual(self.expected(second=0, microsecond=0),
self.ts(func, BarSize.S30))
self.assertEqual(self.expected(second=0, microsecond=0),
self.ts(func, BarSize.M1))
self.assertEqual(self.expected(minute=0, second=0, microsecond=0),
self.ts(func, BarSize.M5))
self.assertEqual(self.expected(minute=0, second=0, microsecond=0),
self.ts(func, BarSize.M15))
self.assertEqual(self.expected(minute=0, second=0, microsecond=0),
self.ts(func, BarSize.M30))
self.assertEqual(self.expected(minute=0, second=0, microsecond=0),
self.ts(func, BarSize.H1))
self.assertEqual(
self.expected(hour=0, minute=0, second=0, microsecond=0),
self.ts(func, BarSize.D1))
def test_current_bar_end_time(self):
func = Bar.get_current_bar_end_time
self.assertEqual(self.expected(second=4, microsecond=999999),
self.ts(func, BarSize.S1))
self.assertEqual(self.expected(second=4, microsecond=999999),
self.ts(func, BarSize.S5))
self.assertEqual(self.expected(second=14, microsecond=999999),
self.ts(func, BarSize.S15))
self.assertEqual(self.expected(second=29, microsecond=999999),
self.ts(func, BarSize.S30))
self.assertEqual(self.expected(second=59, microsecond=999999),
self.ts(func, BarSize.M1))
self.assertEqual(
self.expected(minute=4, second=59, microsecond=999999),
self.ts(func, BarSize.M5))
self.assertEqual(
self.expected(minute=14, second=59, microsecond=999999),
self.ts(func, BarSize.M15))
self.assertEqual(
self.expected(minute=29, second=59, microsecond=999999),
self.ts(func, BarSize.M30))
self.assertEqual(
self.expected(minute=59, second=59, microsecond=999999),
self.ts(func, BarSize.H1))
self.assertEqual(
self.expected(hour=23, minute=59, second=59, microsecond=999999),
self.ts(func, BarSize.D1))
def test_next_bar_start_time(self):
func = Bar.get_next_bar_start_time
self.assertEqual(self.expected(second=5, microsecond=0),
self.ts(func, BarSize.S1))
self.assertEqual(self.expected(second=5, microsecond=0),
self.ts(func, BarSize.S5))
self.assertEqual(self.expected(second=15, microsecond=0),
self.ts(func, BarSize.S15))
self.assertEqual(self.expected(second=30, microsecond=0),
self.ts(func, BarSize.S30))
self.assertEqual(self.expected(minute=4, second=0, microsecond=0),
self.ts(func, BarSize.M1))
self.assertEqual(self.expected(minute=5, second=0, microsecond=0),
self.ts(func, BarSize.M5))
self.assertEqual(self.expected(minute=15, second=0, microsecond=0),
self.ts(func, BarSize.M15))
self.assertEqual(self.expected(minute=30, second=0, microsecond=0),
self.ts(func, BarSize.M30))
self.assertEqual(
self.expected(hour=7, minute=0, second=0, microsecond=0),
self.ts(func, BarSize.H1))
self.assertEqual(
self.expected(day=2, hour=0, minute=0, second=0, microsecond=0),
self.ts(func, BarSize.D1))
def get_current_bar_start_time(timestamp, bar_size):
if bar_size < BarSize.D1:
return int(timestamp / (bar_size * 1000)) * bar_size * 1000
else:
dt = datetime.datetime.fromtimestamp(timestamp / 1000)
return DateUtils.datetime_to_unixtimemillis(datetime.datetime(year=dt.year, month=dt.month, day=dt.day))
