def testResampleNinjaTraderHour(self):
with common.TmpDir() as tmp_path:
# Resample.
feed = ninjatraderfeed.Feed(ninjatraderfeed.Frequency.MINUTE)
feed.addBarsFromCSV("spy", common.get_data_file_path("nt-spy-minute-2011.csv"))
resampledBarDS = resampled_ds.ResampledBarDataSeries(feed["spy"], bar.Frequency.HOUR)
resampledFile = os.path.join(tmp_path, "hour-nt-spy-minute-2011.csv")
resample.resample_to_csv(feed, bar.Frequency.HOUR, resampledFile)
resampledBarDS.pushLast() # Need to manually push the last stot since time didn't change.
# Load the resampled file.
feed = csvfeed.GenericBarFeed(bar.Frequency.HOUR, marketsession.USEquities.getTimezone())
feed.addBarsFromCSV("spy", resampledFile)
feed.loadAll()
self.assertEqual(len(feed["spy"]), 340)
self.assertEqual(feed["spy"][0].getDateTime(), dt.localize(datetime.datetime(2011, 1, 3, 9), marketsession.USEquities.getTimezone()))
self.assertEqual(feed["spy"][-1].getDateTime(), dt.localize(datetime.datetime(2011, 2, 1, 1), marketsession.USEquities.getTimezone()))
self.assertEqual(feed["spy"][0].getOpen(), 126.35)
self.assertEqual(feed["spy"][0].getHigh(), 126.45)
self.assertEqual(feed["spy"][0].getLow(), 126.3)
self.assertEqual(feed["spy"][0].getClose(), 126.4)
self.assertEqual(feed["spy"][0].getVolume(), 3397.0)
self.assertEqual(feed["spy"][0].getAdjClose(), None)
self.assertEqual(len(resampledBarDS), len(feed["spy"]))
self.assertEqual(resampledBarDS[0].getDateTime(), dt.as_utc(datetime.datetime(2011, 1, 3, 9)))
self.assertEqual(resampledBarDS[-1].getDateTime(), dt.as_utc(datetime.datetime(2011, 2, 1, 1)))
def __init__(self, dateTime):
super(MonthRange, self).__init__()
self.__begin = datetime.datetime(dateTime.year, dateTime.month, 1)
# Calculate the ending date.
if dateTime.month == 12:
self.__end = datetime.datetime(dateTime.year + 1, 1, 1)
else:
self.__end = datetime.datetime(dateTime.year, dateTime.month + 1, 1)
if not dt.datetime_is_naive(dateTime):
self.__begin = dt.localize(self.__begin, dateTime.tzinfo)
self.__end = dt.localize(self.__end, dateTime.tzinfo)
def __init__(self, dateTime):
super(MonthRange, self).__init__()
self.__begin = datetime.datetime(dateTime.year, dateTime.month, 1)
# Calculate the ending date.
if dateTime.month == 12:
self.__end = datetime.datetime(dateTime.year + 1, 1, 1)
else:
self.__end = datetime.datetime(dateTime.year, dateTime.month + 1,
1)
if not dt.datetime_is_naive(dateTime):
self.__begin = dt.localize(self.__begin, dateTime.tzinfo)
self.__end = dt.localize(self.__end, dateTime.tzinfo)
def __testHourRangeImpl(self, timezone=None):
freq = bar.Frequency.HOUR
begin = datetime.datetime(2011, 1, 1, 16)
end = datetime.datetime(2011, 1, 1, 17)
if timezone is not None:
begin = dt.localize(begin, timezone)
end = dt.localize(end, timezone)
r = resamplebase.build_range(begin + datetime.timedelta(seconds=120), freq)
self.assertEqual(r.getBeginning(), begin)
for i in range(freq):
self.assertTrue(r.belongs(begin + datetime.timedelta(seconds=i)))
self.assertFalse(r.belongs(begin + datetime.timedelta(seconds=freq+1)))
self.assertEqual(r.getEnding(), end)
def getBars(self,
instrument,
frequency,
timezone=None,
fromDateTime=None,
toDateTime=None):
instrument = normalize_instrument(instrument)
sql = "select bar.timestamp, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.adj_close, bar.frequency" \
" from bar join instrument on (bar.instrument_id = instrument.instrument_id)" \
" where instrument.name = ? and bar.frequency = ?"
args = [instrument, frequency]
if fromDateTime is not None:
sql += " and bar.timestamp >= ?"
args.append(dt.datetime_to_timestamp(fromDateTime))
if toDateTime is not None:
sql += " and bar.timestamp <= ?"
args.append(dt.datetime_to_timestamp(toDateTime))
sql += " order by bar.timestamp asc"
cursor = self.__connection.cursor()
cursor.execute(sql, args)
ret = []
for row in cursor:
dateTime = dt.timestamp_to_datetime(row[0])
if timezone:
dateTime = dt.localize(dateTime, timezone)
ret.append(
bar.BasicBar(dateTime, row[1], row[2], row[3], row[4], row[5],
row[6], row[7]))
cursor.close()
return ret
def testFeedWithQuandl(self):
class RowFilter(csvfeed.RowFilter):
def includeRow(self, dateTime, values):
return dateTime.year == 2013
feed = csvfeed.Feed("Date",
"%Y-%m-%d",
maxLen=40,
timezone=marketsession.USEquities.timezone)
feed.setRowFilter(RowFilter())
feed.setTimeDelta(datetime.timedelta(hours=23, minutes=59, seconds=59))
feed.addValuesFromCSV(
os.path.join("samples", "data", "quandl_gold_2.csv"))
for col in ["USD", "GBP", "EUR"]:
self.assertEqual(len(feed[col]), 0)
disp = dispatcher.Dispatcher()
disp.addSubject(feed)
disp.run()
for col in ["USD", "GBP", "EUR"]:
self.assertEqual(len(feed[col]), 39)
self.assertEqual(feed["USD"][-1], 1333.0)
self.assertEqual(feed["GBP"][-1], 831.203)
self.assertEqual(feed["EUR"][-1], 986.75)
self.assertFalse(dt.datetime_is_naive(feed["USD"].getDateTimes()[-1]))
self.assertEqual(
feed["USD"].getDateTimes()[-1],
dt.localize(datetime.datetime(2013, 9, 29, 23, 59, 59),
marketsession.USEquities.timezone))
def testFeedWithQuandl(self):
class RowFilter(csvfeed.RowFilter):
def includeRow(self, dateTime, values):
return dateTime.year == 2013
feed = csvfeed.Feed("Date", "%Y-%m-%d", maxLen=40, timezone=marketsession.USEquities.timezone)
feed.setRowFilter(RowFilter())
feed.setTimeDelta(datetime.timedelta(hours=23, minutes=59, seconds=59))
feed.addValuesFromCSV(os.path.join("samples", "data", "quandl_gold_2.csv"))
for col in ["USD", "GBP", "EUR"]:
self.assertEqual(len(feed[col]), 0)
disp = dispatcher.Dispatcher()
disp.addSubject(feed)
disp.run()
for col in ["USD", "GBP", "EUR"]:
self.assertEqual(len(feed[col]), 39)
self.assertEqual(feed["USD"][-1], 1333.0)
self.assertEqual(feed["GBP"][-1], 831.203)
self.assertEqual(feed["EUR"][-1], 986.75)
self.assertFalse(dt.datetime_is_naive(feed["USD"].getDateTimes()[-1]))
self.assertEqual(
feed["USD"].getDateTimes()[-1],
dt.localize(datetime.datetime(2013, 9, 29, 23, 59, 59), marketsession.USEquities.timezone)
)
def to_market_datetime(dateTime, exchange):
timezone = MARKET_TIMEZONES.get(exchange)
if timezone is None:
raise Exception(
"No timezone available to localize datetime for exchange %s" %
(exchange))
return dt.localize(dateTime, timezone)
def __init__(self, dateTime):
super(DayRange, self).__init__()
self.__begin = datetime.datetime(dateTime.year, dateTime.month,
dateTime.day)
if not dt.datetime_is_naive(dateTime):
self.__begin = dt.localize(self.__begin, dateTime.tzinfo)
self.__end = self.__begin + datetime.timedelta(days=1)
def getBars(self, instrument, frequency, timezone=None, fromDateTime=None, toDateTime=None):
instrument = normalize_instrument(instrument)
sql = "select bar.timestamp, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.adj_close, bar.frequency" \
" from bar join instrument on (bar.instrument_id = instrument.instrument_id)" \
" where instrument.name = ? and bar.frequency = ?"
args = [instrument, frequency]
if fromDateTime is not None:
sql += " and bar.timestamp >= ?"
args.append(dt.datetime_to_timestamp(fromDateTime))
if toDateTime is not None:
sql += " and bar.timestamp <= ?"
args.append(dt.datetime_to_timestamp(toDateTime))
sql += " order by bar.timestamp asc"
cursor = self.__connection.cursor()
cursor.execute(sql, args)
ret = []
for row in cursor:
dateTime = dt.timestamp_to_datetime(row[0])
if timezone:
dateTime = dt.localize(dateTime, timezone)
ret.append(bar.BasicBar(dateTime, row[1], row[2], row[3], row[4], row[5], row[6], row[7]))
cursor.close()
return ret
def __parseDate(self, dateString):
ret = parse_date(dateString)
# Time on Yahoo! Finance CSV files is empty. If told to set one, do it.
if self.__dailyBarTime is not None:
ret = datetime.datetime.combine(ret, self.__dailyBarTime)
# Localize the datetime if a timezone was given.
if self.__timezone:
ret = dt.localize(ret, self.__timezone)
return ret
def __parseDate(self, dateString):
ret = parse_date(dateString)
# Time on Yahoo! Finance CSV files is empty. If told to set one, do it.
if self.__dailyBarTime is not None:
ret = datetime.datetime.combine(ret, self.__dailyBarTime)
# Localize the datetime if a timezone was given.
if self.__timezone:
ret = dt.localize(ret, self.__timezone)
return ret
def parseBar(self, csvRowDict):
unixTime = int(csvRowDict["unixtime"])
price = float(csvRowDict["price"])
amount = float(csvRowDict["amount"])
dateTime = dt.timestamp_to_datetime(unixTime)
dateTime = self.__unixTimeFix.fixDateTime(dateTime)
# Localize the datetime if a timezone was given.
if self.__timezone:
dateTime = dt.localize(dateTime, self.__timezone)
return TradeBar(dateTime, price, amount)
def parseRow(self, csvRowDict):
dateTime = datetime.datetime.strptime(csvRowDict[self.__dateTimeColumn], self.__dateTimeFormat)
# Localize the datetime if a timezone was given.
if self.__timezone is not None:
if self.__timeDelta is not None:
dateTime += self.__timeDelta
dateTime = dt.localize(dateTime, self.__timezone)
# Convert the values
values = {}
for key, value in csvRowDict.items():
if key != self.__dateTimeColumn:
values[key] = self.__converter(key, value)
return (dateTime, values)
def parseBar(self, csvRowDict):
unixTime = int(csvRowDict["unixtime"])
price = float(csvRowDict["price"])
amount = float(csvRowDict["amount"])
dateTime = dt.timestamp_to_datetime(unixTime)
dateTime = self.__unixTimeFix.fixDateTime(dateTime)
# Localize the datetime if a timezone was given.
if self.__timezone:
dateTime = dt.localize(dateTime, self.__timezone)
return TradeBar(dateTime, price, amount)
def __init__(self, dateTime, frequency):
super(IntraDayRange, self).__init__()
assert isinstance(frequency, int)
assert frequency > 1
assert frequency < bar.Frequency.DAY
ts = int(dt.datetime_to_timestamp(dateTime))
slot = int(ts / frequency)
slotTs = slot * frequency
self.__begin = dt.timestamp_to_datetime(slotTs, not dt.datetime_is_naive(dateTime))
if not dt.datetime_is_naive(dateTime):
self.__begin = dt.localize(self.__begin, dateTime.tzinfo)
self.__end = self.__begin + datetime.timedelta(seconds=frequency)
def testResampleNinjaTraderDay(self):
with common.TmpDir() as tmp_path:
# Resample.
feed = ninjatraderfeed.Feed(ninjatraderfeed.Frequency.MINUTE)
feed.addBarsFromCSV("spy", common.get_data_file_path("nt-spy-minute-2011.csv"))
resampledBarDS = resampled_ds.ResampledBarDataSeries(feed["spy"], bar.Frequency.DAY)
resampledFile = os.path.join(tmp_path, "day-nt-spy-minute-2011.csv")
resample.resample_to_csv(feed, bar.Frequency.DAY, resampledFile)
resampledBarDS.pushLast() # Need to manually push the last stot since time didn't change.
# Load the resampled file.
feed = csvfeed.GenericBarFeed(bar.Frequency.DAY)
feed.addBarsFromCSV("spy", resampledFile, marketsession.USEquities.getTimezone())
feed.loadAll()
self.assertEqual(len(feed["spy"]), 25)
self.assertEqual(feed["spy"][0].getDateTime(), dt.localize(datetime.datetime(2011, 1, 3), marketsession.USEquities.getTimezone()))
self.assertEqual(feed["spy"][-1].getDateTime(), dt.localize(datetime.datetime(2011, 2, 1), marketsession.USEquities.getTimezone()))
self.assertEqual(len(resampledBarDS), len(feed["spy"]))
self.assertEqual(resampledBarDS[0].getDateTime(), dt.as_utc(datetime.datetime(2011, 1, 3)))
self.assertEqual(resampledBarDS[-1].getDateTime(), dt.as_utc(datetime.datetime(2011, 2, 1)))
def __init__(self, dateTime, frequency):
super(IntraDayRange, self).__init__()
assert isinstance(frequency, int)
assert frequency > 1
assert frequency < bar.Frequency.DAY
ts = int(dt.datetime_to_timestamp(dateTime))
slot = int(ts / frequency)
slotTs = slot * frequency
self.__begin = dt.timestamp_to_datetime(
slotTs, not dt.datetime_is_naive(dateTime))
if not dt.datetime_is_naive(dateTime):
self.__begin = dt.localize(self.__begin, dateTime.tzinfo)
self.__end = self.__begin + datetime.timedelta(seconds=frequency)
def parseRow(self, csvRowDict):
dateTime = datetime.datetime.strptime(
csvRowDict[self.__dateTimeColumn], self.__dateTimeFormat)
# Localize the datetime if a timezone was given.
if self.__timezone is not None:
if self.__timeDelta is not None:
dateTime += self.__timeDelta
dateTime = dt.localize(dateTime, self.__timezone)
# Convert the values
values = {}
for key, value in csvRowDict.items():
if key != self.__dateTimeColumn:
values[key] = self.__converter(key, value)
return (dateTime, values)
def __parseDateTime(self, dateTime):
ret = None
if self.__frequency == engine.bar.Frequency.MINUTE:
ret = parse_datetime(dateTime)
elif self.__frequency == engine.bar.Frequency.DAY:
ret = datetime.datetime.strptime(dateTime, "%Y%m%d")
# Time on CSV files is empty. If told to set one, do it.
if self.__dailyBarTime is not None:
ret = datetime.datetime.combine(ret, self.__dailyBarTime)
else:
assert(False)
# According to NinjaTrader documentation the exported data will be in UTC.
ret = pytz.utc.localize(ret)
# Localize bars if a market session was set.
if self.__timezone:
ret = dt.localize(ret, self.__timezone)
return ret
def __parseDateTime(self, dateTime):
ret = None
if self.__frequency == engine.bar.Frequency.MINUTE:
ret = parse_datetime(dateTime)
elif self.__frequency == engine.bar.Frequency.DAY:
ret = datetime.datetime.strptime(dateTime, "%Y%m%d")
# Time on CSV files is empty. If told to set one, do it.
if self.__dailyBarTime is not None:
ret = datetime.datetime.combine(ret, self.__dailyBarTime)
else:
assert (False)
# According to NinjaTrader documentation the exported data will be in UTC.
ret = pytz.utc.localize(ret)
# Localize bars if a market session was set.
if self.__timezone:
ret = dt.localize(ret, self.__timezone)
return ret
def testLocalizeAndFilter(self):
timezone = marketsession.USEquities.getTimezone()
# The prices come from NinjaTrader interface when set to use 'US Equities RTH' session template.
prices = {
dt.localize(datetime.datetime(2011, 3, 9, 9, 31), timezone): 132.35,
dt.localize(datetime.datetime(2011, 3, 9, 16), timezone): 132.39,
dt.localize(datetime.datetime(2011, 3, 10, 9, 31), timezone): 130.81,
dt.localize(datetime.datetime(2011, 3, 10, 16), timezone): 129.92,
dt.localize(datetime.datetime(2011, 3, 11, 9, 31), timezone): 129.72,
dt.localize(datetime.datetime(2011, 3, 11, 16), timezone): 130.84,
}
barFeed = ninjatraderfeed.Feed(ninjatraderfeed.Frequency.MINUTE, timezone)
barFeed.addBarsFromCSV("spy", common.get_data_file_path("nt-spy-minute-2011-03.csv"))
for dateTime, bars in barFeed:
price = prices.get(bars.getDateTime(), None)
if price is not None:
self.assertTrue(price == bars.getBar("spy").getClose())
def to_market_datetime(dateTime):
timezone = pytz.timezone('Asia/Shanghai')
return dt.localize(dateTime, timezone)
def to_market_datetime(dateTime):
timezone = pytz.timezone('Asia/Shanghai')
return dt.localize(dateTime, timezone)
def to_market_datetime(dateTime, exchange):
timezone = MARKET_TIMEZONES.get(exchange)
if timezone is None:
raise Exception("No timezone available to localize datetime for exchange %s" % (exchange))
return dt.localize(dateTime, timezone)
def __init__(self, dateTime):
super(DayRange, self).__init__()
self.__begin = datetime.datetime(dateTime.year, dateTime.month, dateTime.day)
if not dt.datetime_is_naive(dateTime):
self.__begin = dt.localize(self.__begin, dateTime.tzinfo)
self.__end = self.__begin + datetime.timedelta(days=1)