def testBaseBarFeedNoAdjClose(self):
bars = [
bar.Bars({"orcl": bar.BasicBar(datetime.datetime(2001, 1, 1), 1, 1, 1, 1, 1, None, bar.Frequency.DAY)}),
bar.Bars({"orcl": bar.BasicBar(datetime.datetime(2001, 1, 2), 1, 1, 1, 1, 1, None, bar.Frequency.DAY)}),
]
barFeed = barfeed.OptimizerBarFeed(bar.Frequency.DAY, ["orcl"], bars)
check_base_barfeed(self, barFeed, False)
def testResample(self):
barDs = bards.BarDataSeries()
resampledBarDS = resampled.ResampledBarDataSeries(
barDs, barfeed.Frequency.MINUTE)
barDs.append(
bar.BasicBar(datetime.datetime(2011, 1, 1, 1, 1, 1), 2.1, 3, 1, 2,
10, 1))
barDs.append(
bar.BasicBar(datetime.datetime(2011, 1, 1, 1, 1, 2), 2, 3, 1, 2.3,
10, 2))
barDs.append(
bar.BasicBar(datetime.datetime(2011, 1, 1, 1, 2, 1), 2, 3, 1, 2,
10, 2))
self.assertEqual(len(resampledBarDS), 1)
self.assertEqual(resampledBarDS[0].getDateTime(),
datetime.datetime(2011, 1, 1, 1, 1, 59))
self.assertEqual(resampledBarDS[0].getOpen(), 2.1)
self.assertEqual(resampledBarDS[0].getHigh(), 3)
self.assertEqual(resampledBarDS[0].getLow(), 1)
self.assertEqual(resampledBarDS[0].getClose(), 2.3)
self.assertEqual(resampledBarDS[0].getVolume(), 20)
self.assertEqual(resampledBarDS[0].getAdjClose(), 2)
resampledBarDS.pushLast()
self.assertEqual(len(resampledBarDS), 2)
self.assertEqual(resampledBarDS[1].getDateTime(),
datetime.datetime(2011, 1, 1, 1, 2, 59))
self.assertEqual(resampledBarDS[1].getOpen(), 2)
self.assertEqual(resampledBarDS[1].getHigh(), 3)
self.assertEqual(resampledBarDS[1].getLow(), 1)
self.assertEqual(resampledBarDS[1].getClose(), 2)
self.assertEqual(resampledBarDS[1].getVolume(), 10)
self.assertEqual(resampledBarDS[1].getAdjClose(), 2)
def testInvalidDateTimes(self):
b1 = bar.BasicBar(datetime.datetime.now(), 2, 3, 1, 2.1, 10, 5,
bar.Frequency.DAY)
b2 = bar.BasicBar(datetime.datetime.now() + datetime.timedelta(days=1),
2, 3, 1, 2.1, 10, 5, bar.Frequency.DAY)
with self.assertRaises(Exception):
bar.Bars({"a": b1, "b": b2})
def testDateTimesNotInOrder(self):
bars = [
bar.Bars({"orcl": bar.BasicBar(datetime.datetime(2001, 1, 2), 1, 1, 1, 1, 1, 1, bar.Frequency.DAY)}),
bar.Bars({"orcl": bar.BasicBar(datetime.datetime(2001, 1, 1), 1, 1, 1, 1, 1, 1, bar.Frequency.DAY)}),
]
f = barfeed.OptimizerBarFeed(bar.Frequency.DAY, ["orcl"], bars)
with self.assertRaisesRegexp(Exception, "Bar date times are not in order.*"):
for dt, b in f:
pass
def testInvalidDateTimes(self):
b1 = bar.BasicBar(INSTRUMENT, datetime.datetime.now(), 2, 3, 1, 2.1,
10, 5, bar.Frequency.DAY)
b2 = bar.BasicBar(INSTRUMENT,
datetime.datetime.now() + datetime.timedelta(days=1),
2, 3, 1, 2.1, 10, 5, bar.Frequency.DAY)
with self.assertRaisesRegexp(Exception,
"Bar data times are not in sync"):
bar.Bars([b1, b2])
def testAppendInvalidDatetime(self):
ds = bards.BarDataSeries()
for i in range(10):
now = datetime.datetime.now() + datetime.timedelta(seconds=i)
ds.append(bar.BasicBar(now, 0, 0, 0, 0, 0, 0, bar.Frequency.SECOND))
# Adding the same datetime twice should fail
self.assertRaises(Exception, ds.append, bar.BasicBar(now, 0, 0, 0, 0, 0, 0, bar.Frequency.SECOND))
# Adding a previous datetime should fail
self.assertRaises(Exception, ds.append, bar.BasicBar(now - datetime.timedelta(seconds=i), 0, 0, 0, 0, 0, 0, bar.Frequency.SECOND))
def generateBars(self):
if self.__pullDelay > 0:
time.sleep(self.__pullDelay)
logger.info('get quote of {0}'.format(self.__instrument))
tmp = quote(self.__instrument)
if tmp is None:
logger.error('failed to get {0} quote'.format(self.__instrument))
return
tmpVal = tmp.iloc[0]['close'].iloc[0]
curTime = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
tmp = bar.BasicBar(curTime,
tmpVal, tmpVal, tmpVal, tmpVal,
0, False, Frequency.REALTIME)
for freq in self.__nextRealtimeBars.keys():
if self.__nextRealtimeBars[freq]['open'] is None:
self.__nextRealtimeBars[freq]['open'] = tmpVal
self.__nextRealtimeBars[freq]['high'] = tmpVal
self.__nextRealtimeBars[freq]['low'] = tmpVal
self.__nextRealtimeBars[freq]['close'] = tmpVal
if freq == Frequency.DAY:
curTime = curTime.replace(hour=0, minute=0, second=0, microsecond=0)
elif freq == Frequency.HOUR:
curTime = curTime.replace(minute=0, second=0, microsecond=0)
elif freq == Frequency.MINUTE:
curTime = curTime.replace(second=0, microsecond=0)
elif freq != Frequency.REALTIME:
logger.error('{0} is not supported.'.format(freq))
assert False
self.__nextRealtimeBars[freq]['start'] = curTime
else:
if tmpVal > self.__nextRealtimeBars[freq]['high']:
self.__nextRealtimeBars[freq]['high'] = tmpVal
elif tmpVal < self.__nextRealtimeBars[freq]['low']:
self.__nextRealtimeBars[freq]['low'] = tmpVal
deltaTime = curTime - self.__nextRealtimeBars[freq]['start']
if (Frequency.MINUTE == freq and deltaTime.total_seconds() > 60 or
Frequency.DAY == freq and deltaTime.total_seconds() > 60*60*24 or
Frequency.HOUR == freq and deltaTime.total_seconds() > 60*60):
self.__nextRealtimeBars[freq]['close'] = tmpVal
row = self.__nextRealtimeBars[freq]
tmpbar = bar.BasicBar(curTime, row['open'], row['high'],
row['low'], row['close'], 0, False,
freq)
self.__bars_buf.append(tmpbar)
self.__nextRealtimeBars[freq] = {
'open': None,
'high': None,
'low': None,
'close': None,
'start' : None,
}
self.__bars_buf.append(tmp)
if self.__isRealTime:
self.__bars_buf.append(tmp)
self.__bars_buf.sort(key=lambda i: i.getDateTime())
def testIter(self):
dt = datetime.datetime.now()
b1 = bar.BasicBar("a/USD", dt, 1, 1, 1, 1, 10, 1, bar.Frequency.DAY)
b2 = bar.BasicBar("a/EUR", dt, 2, 2, 2, 2, 10, 2, bar.Frequency.DAY)
b3 = bar.BasicBar("b/USD", dt, 2, 2, 2, 2, 10, 2, bar.Frequency.DAY)
bars = bar.Bars([b1, b2, b3])
items = [b1, b2, b3]
for item in bars:
items.remove(item)
assert len(items) == 0
def main():
bf = TestBarFeed(bar.Frequency.DAY)
bars = [
bar.BasicBar(datetime.datetime(2000, 1, 1), 10, 10, 10, 10, 10, 10, bar.Frequency.DAY),
bar.BasicBar(datetime.datetime(2000, 1, 2), 10, 10, 10, 10, 10, 10, bar.Frequency.DAY),
]
bf.addBarsFromSequence("orcl", bars)
logger.getLogger().setLevel(logging.DEBUG)
strat = BacktestingStrategy(bf, 1)
strat.run()
def testInvalidConstruction(self):
with self.assertRaises(Exception):
bar.BasicBar(datetime.datetime.now(), 2, 1, 1, 1, 1, 1, bar.Frequency.DAY)
with self.assertRaises(Exception):
bar.BasicBar(datetime.datetime.now(), 1, 1, 1, 2, 1, 1, bar.Frequency.DAY)
with self.assertRaises(Exception):
bar.BasicBar(datetime.datetime.now(), 1, 2, 1.5, 1, 1, 1, bar.Frequency.DAY)
with self.assertRaises(Exception):
bar.BasicBar(datetime.datetime.now(), 2, 2, 1.5, 1, 1, 1, bar.Frequency.DAY)
with self.assertRaises(Exception):
bar.BasicBar(datetime.datetime.now(), 1, 1, 1.5, 1, 1, 1, bar.Frequency.DAY)
def testBasic(self):
dt = datetime.datetime.now()
b1 = bar.BasicBar(dt, 1, 1, 1, 1, 10, 1, bar.Frequency.DAY)
b2 = bar.BasicBar(dt, 2, 2, 2, 2, 10, 2, bar.Frequency.DAY)
bars = bar.Bars({"a": b1, "b": b2})
self.assertEqual(bars["a"].getClose(), 1)
self.assertEqual(bars["b"].getClose(), 2)
self.assertTrue("a" in bars)
self.assertEqual(bars.items(), [("a", b1), ("b", b2)])
self.assertEqual(bars.keys(), ["a", "b"])
self.assertEqual(bars.getInstruments(), ["a", "b"])
self.assertEqual(bars.getDateTime(), dt)
self.assertEqual(bars.getBar("a").getClose(), 1)
def testBaseBarFeedNoAdjClose(self):
bars = [
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 1, None, bar.Frequency.DAY)
]),
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 2), 1, 1,
1, 1, 1, None, bar.Frequency.DAY)
]),
]
barFeed = barfeed.OptimizerBarFeed(bar.Frequency.DAY, [INSTRUMENT],
bars)
check_base_barfeed(self, barFeed, False)
def testBlackWhiteReversal(self):
bds = bards.BarDataSeries()
lb = linebreak.LineBreak(bds, 2)
bds.append(bar.BasicBar(datetime.datetime(2008, 3, 5), 10, 12, 8, 9, 1, None, bar.Frequency.DAY))
self.assertEqual(len(lb), 1)
bds.append(bar.BasicBar(datetime.datetime(2008, 3, 6), 9, 12, 9, 12, 1, None, bar.Frequency.DAY))
self.assertEqual(len(lb), 1)
self.assertEqual(lb[-1].isBlack(), True)
self.assertEqual(lb[-1].getDateTime(), datetime.datetime(2008, 3, 5))
bds.append(bar.BasicBar(datetime.datetime(2008, 3, 7), 9, 13, 5, 13, 1, None, bar.Frequency.DAY))
self.assertEqual(len(lb), 2)
self.assertEqual(lb[-1].isWhite(), True)
self.assertEqual(lb[-1].getDateTime(), datetime.datetime(2008, 3, 7))
def testBasicOperations(self):
dt = datetime.datetime.now()
b1 = bar.BasicBar("a/usd", dt, 1, 1, 1, 1, 10, 1, bar.Frequency.DAY)
b2 = bar.BasicBar("b/usd", dt, 2, 2, 2, 2, 10, 2, bar.Frequency.DAY)
bars = bar.Bars([b1, b2])
self.assertEqual(bars["a/usd"].getClose(), 1)
self.assertEqual(bars["b/usd"].getClose(), 2)
self.assertTrue("a/usd" in bars)
self.assertEqual(bars.getInstruments(), ["a/usd", "b/usd"])
self.assertEqual(bars.getDateTime(), dt)
self.assertEqual(bars.getBar("a/usd").getClose(), 1)
with self.assertRaises(KeyError):
bars["c/usd"]
def build_bar(barDict, identifier, frequency):
# "StartDate": "3/19/2014"
# "StartTime": "9:55:00 AM"
# "EndDate": "3/19/2014"
# "EndTime": "10:00:00 AM"
# "UTCOffset": 0
# "Open": 31.71
# "High": 31.71
# "Low": 31.68
# "Close": 31.69
# "Volume": 2966
# "Trades": 19
# "TWAP": 31.6929
# "VWAP": 31.693
startDate = barDict["StartDate"]
startTime = barDict["StartTime"]
startDateTimeStr = startDate + " " + startTime
startDateTime = datetime.datetime.strptime(startDateTimeStr,
"%m/%d/%Y %I:%M:%S %p")
instrument, exchange = api.parse_instrument_exchange(identifier)
startDateTime = api.to_market_datetime(startDateTime, exchange)
return bar.BasicBar(startDateTime, barDict["Open"], barDict["High"],
barDict["Low"], barDict["Close"], barDict["Volume"],
None, frequency)
def getBars(self,
instrument,
frequency,
timezone=None,
fromDateTime=None,
toDateTime=None):
sql = "select fecha, valor" \
" from dato where activo = %s and criterio = %s"
args = [instrument, self.__priceField]
if fromDateTime is not None:
sql += " and fecha >= %s"
args.append(fromDateTime)
if toDateTime is not None:
sql += " and fecha <= %s"
args.append(toDateTime)
sql += " order by fecha asc"
cursor = self.__connection.cursor()
cursor.execute(sql, args)
ret = []
for dateTime, value in cursor:
if timezone:
dateTime = dt.localize(dateTime, timezone)
ret.append(
bar.BasicBar(dateTime, value, value, value, value, 0, value,
frequency))
cursor.close()
return ret
def __buildBar(self, openPrice, highPrice, lowPrice, closePrice):
dateTime = datetime.datetime.now() + datetime.timedelta(
seconds=self.__currSeconds)
self.__currSeconds += 1
return bar.BasicBar(dateTime, openPrice, highPrice, lowPrice,
closePrice, closePrice * 10, closePrice,
bar.Frequency.DAY)
def build_bar(barDict, frequency):
# "StartDate": "3/19/2014"
# "StartTime": "9:55:00 AM"
# "EndDate": "3/19/2014"
# "EndTime": "10:00:00 AM"
# "UTCOffset": 0
# "Open": 31.71
# "High": 31.71
# "Low": 31.68
# "Close": 31.69
# "Volume": 2966
# "Trades": 19
# "TWAP": 31.6929
# "VWAP": 31.693
startDateTimeStr = barDict["date"].values()[0]
if len(startDateTimeStr) == 10:
startDateTime = datetime.datetime.strptime(startDateTimeStr,
"%Y-%m-%d")
else:
startDateTime = datetime.datetime.strptime(startDateTimeStr[:16],
'%Y-%m-%d %H:%M')
return bar.BasicBar(startDateTime, barDict["open"].values()[0],
barDict["high"].values()[0],
barDict["low"].values()[0],
barDict["close"].values()[0],
barDict["volume"].values()[0], None, frequency)
def get_bar_list(df, frequency, date=None):
bar_list = []
end_time = df.ix[0].time
if date is None:
date = datetime.datetime.now()
slice_start_time = to_market_datetime(
datetime.datetime(date.year, date.month, date.day, 9, 30, 0))
while slice_start_time.strftime("%H:%M:%S") < end_time:
slice_end_time = slice_start_time + datetime.timedelta(
seconds=frequency)
ticks_slice = df.ix[(df.time < slice_end_time.strftime("%H:%M:%S")) &
(df.time >= slice_start_time.strftime("%H:%M:%S"))]
if not ticks_slice.empty:
open_ = ticks_slice.price.get_values()[-1]
high = max(ticks_slice.price)
low = min(ticks_slice.price)
close = ticks_slice.price.get_values()[0]
volume = sum(ticks_slice.volume)
amount = sum(ticks_slice.amount)
bar_list.append(
bar.BasicBar(slice_start_time, open_, high, low, close, volume,
0, frequency, amount))
else:
bar_list.append(None)
slice_start_time = slice_end_time
return bar_list
def getNextBars(self):
if not self.__data_downloaded:
logger.info('featch history of {0}'.format(self.__instrument))
for i in self.getFrequencies():
tmp = None
if i == Frequency.DAY:
tmp = history(self.__instrument, self.__start_date, None,
Frequency.DAY, add_missing_dates=False)
elif i == Frequency.HOUR:
tmp = history(self.__instrument, self.__start_date, None,
Frequency.HOUR, add_missing_dates=False)
if tmp is None:
continue
for date, row in tmp.iloc[0].iterrows():
tmpbar = bar.BasicBar(date, row['open'], row['high'],
row['low'], row['close'], 0, False,
i)
self.__bars_buf.append(tmpbar)
self.__data_downloaded = True
self.__bars_buf.sort(key=lambda i: i.getDateTime())
while True:
if self.__bars_buf:
tmp = self.__bars_buf.pop(0)
return bar.Bars({self.__instrument: tmp}, frequecy=tmp.getFrequency())
else:
self.generateBars()
def parseBar(self, csvRowDict):
dateTime = self._parseDate(csvRowDict[self.__dateTimeColName])
open_ = float(csvRowDict[self.__openColName])
high = float(csvRowDict[self.__highColName])
low = float(csvRowDict[self.__lowColName])
close = float(csvRowDict[self.__closeColName])
volume = float(csvRowDict[self.__volumeColName])
adjClose = None
if self.__adjCloseColName is not None:
adjCloseValue = csvRowDict.get(self.__adjCloseColName, "")
if len(adjCloseValue) > 0:
adjClose = float(adjCloseValue)
self.__haveAdjClose = True
# Process extra columns.
extra = {}
for k, v in csvRowDict.iteritems():
if k not in self.__columnNames:
extra[k] = csvutils.float_or_string(v)
return bar.BasicBar(dateTime,
open_,
high,
low,
close,
volume,
adjClose,
self.__frequency,
extra=extra)
def testDupliateDateTimesForTradeBars(self):
bars = [
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 1, 1, bar.Frequency.TRADE)
]),
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 2, 1, bar.Frequency.TRADE)
]),
]
f = barfeed.OptimizerBarFeed(bar.Frequency.TRADE, [INSTRUMENT], bars)
expected_volume = 1
for dt, b in f:
assert b[INSTRUMENT].getVolume() == expected_volume
expected_volume += 1
def testDupliateDateTimesForDailyBars(self):
bars = [
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 1, 1, bar.Frequency.DAY)
]),
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 1, 1, bar.Frequency.DAY)
]),
]
f = barfeed.OptimizerBarFeed(bar.Frequency.DAY, [INSTRUMENT], bars)
with self.assertRaisesRegexp(Exception, "%s bars are not in order.*" %
INSTRUMENT):
for dt, b in f:
pass
def getBars(self,
instrument,
frequency,
timezone=None,
fromDateTime=None,
toDateTime=None):
instrument = normalize_instrument(instrument)
sql = "SELECT date_mkt, price_open, price_max, price_min, price_close, cast(volume as unsigned) as volume, 0, 0 FROM cota_hist where symbol=%s"
args = [instrument]
if fromDateTime is not None:
sql += " and cota_hist.date_mkt >= ?"
args.append(dt.datetime_to_timestamp(fromDateTime))
if toDateTime is not None:
sql += " and cota_hist.date_mkt <= ?"
args.append(dt.datetime_to_timestamp(toDateTime))
sql += " order by cota_hist.date_mkt asc"
cursor = self.__connection.cursor()
cursor.execute(sql, [instrument])
ret = []
for row in cursor:
#dateTime = dt.timestamp_to_datetime(row[0])
dateTime = row[0]
#if timezone:
# dateTime = dt.localize(dateTime, timezone)
ret.append(
bar.BasicBar(dateTime, float(row[1]), float(row[2]),
float(row[3]), float(row[4]), row[5], row[6],
row[7]))
cursor.close()
return ret
def getGrouped(self):
"""Return the grouped value."""
ret = bar.BasicBar(self.getDateTime(), self.__open, self.__high,
self.__low, self.__close, self.__volume,
self.__adjClose, self.__frequency)
ret.setUseAdjustedValue(self.__useAdjValue)
return ret
def getBars(self,
instrument,
frequency,
bar_count,
fromDateTime=None,
toDateTime=None):
instrument = normalize_instrument(instrument)
# print(instrument, frequency, fromDateTime, toDateTime)
sql = 'SELECT * FROM ' \
'(SELECT quote.* FROM quote, symbol ' \
'WHERE quote.symbol_id = symbol.id and symbol.ticker = %s ORDER BY quote.id DESC limit %s) '
args = [instrument, bar_count]
# if fromDateTime is not None:
# sql += " and bar.timestamp >= ?"
# args.append(dt.datetime_to_timestamp(fromDateTime))
# if toDateTime is not None:
# sql += " and bar.timestamp <= ?"
# args.append(dt.datetime_to_timestamp(toDateTime))
sql += ' sub ORDER BY id ASC'
cursor = self.__connection.cursor()
cursor.execute(sql, args)
quotes = cursor.fetchall()
ret = []
for row in quotes:
bar_data = bar.BasicBar(row[1], row[2], row[3], row[4], row[5],
10000 if row[6] is 0 else row[6], row[5],
None)
ret.append(bar_data)
cursor.close()
return ret
def testNonEmpty(self):
ds = bards.BarDataSeries()
for i in xrange(10):
ds.append(bar.BasicBar(datetime.datetime.now() + datetime.timedelta(seconds=i), 0, 0, 0, 0, 0, 0, bar.Frequency.SECOND))
for i in xrange(0, 10):
self.assertTrue(ds[i].getOpen() == 0)
def getBars(self,
instrument,
frequency,
timezone=None,
fromDateTime=None,
toDateTime=None):
instrument = normalize_instrument(instrument)
sql = "select bar.timestamp, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.adj_close, bar.frequency" \
" from bar join instrument on (bar.instrument_id = instrument.instrument_id)" \
" where instrument.name = ? and bar.frequency = ?"
args = [instrument, frequency]
if fromDateTime is not None:
sql += " and bar.timestamp >= ?"
args.append(dt.datetime_to_timestamp(fromDateTime))
if toDateTime is not None:
sql += " and bar.timestamp <= ?"
args.append(dt.datetime_to_timestamp(toDateTime))
sql += " order by bar.timestamp asc"
cursor = self.__connection.cursor()
cursor.execute(sql, args)
ret = []
for row in cursor:
dateTime = dt.timestamp_to_datetime(row[0])
if timezone:
dateTime = dt.localize(dateTime, timezone)
ret.append(
bar.BasicBar(dateTime, row[1], row[2], row[3], row[4], row[5],
row[6], row[7]))
cursor.close()
return ret
def buildBars(self, openPrice, highPrice, lowPrice, closePrice, sessionClose = False):
ret = {}
dateTime = self.__getNextDateTime(sessionClose)
bar_ = bar.BasicBar(dateTime, openPrice, highPrice, lowPrice, closePrice, closePrice*10, closePrice)
bar_.setSessionClose(sessionClose)
ret[BaseTestCase.TestInstrument] = bar_
return bar.Bars(ret)
def parseBar(self, csvRowDict):
dateTime = self.__parseDateTime(csvRowDict["Date Time"])
close = float(csvRowDict["Close"])
open_ = float(csvRowDict["Open"])
high = float(csvRowDict["High"])
low = float(csvRowDict["Low"])
volume = float(csvRowDict["Volume"])
return bar.BasicBar(dateTime, open_, high, low, close, volume, None, self.__frequency)
