def testBaseBarFeedNoAdjClose(self):
bars = [
bar.Bars({"orcl": bar.BasicBar(datetime.datetime(2001, 1, 1), 1, 1, 1, 1, 1, None, bar.Frequency.DAY)}),
bar.Bars({"orcl": bar.BasicBar(datetime.datetime(2001, 1, 2), 1, 1, 1, 1, 1, None, bar.Frequency.DAY)}),
]
barFeed = barfeed.OptimizerBarFeed(bar.Frequency.DAY, ["orcl"], bars)
check_base_barfeed(self, barFeed, False)
def testDateTimesNotInOrder(self):
bars = [
bar.Bars({"orcl": bar.BasicBar(datetime.datetime(2001, 1, 2), 1, 1, 1, 1, 1, 1, bar.Frequency.DAY)}),
bar.Bars({"orcl": bar.BasicBar(datetime.datetime(2001, 1, 1), 1, 1, 1, 1, 1, 1, bar.Frequency.DAY)}),
]
f = barfeed.OptimizerBarFeed(bar.Frequency.DAY, ["orcl"], bars)
with self.assertRaisesRegexp(Exception, "Bar date times are not in order.*"):
for dt, b in f:
pass
def testBaseBarFeedNoAdjClose(self):
bars = [
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 1, None, bar.Frequency.DAY)
]),
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 2), 1, 1,
1, 1, 1, None, bar.Frequency.DAY)
]),
]
barFeed = barfeed.OptimizerBarFeed(bar.Frequency.DAY, [INSTRUMENT],
bars)
check_base_barfeed(self, barFeed, False)
def getNextBars(self):
# All bars must have the same datetime. We will return all the ones with the smallest datetime.
smallestDateTime = self.peekDateTime()
if smallestDateTime is None:
assert (self.__barsLeft == 0)
return None
# Make a second pass to get all the bars that had the smallest datetime.
ret = {}
for instrument, bars in self.__bars.iteritems():
nextIdx = self.__nextBarIdx[instrument]
if nextIdx < len(bars) and bars[nextIdx].getDateTime(
) == smallestDateTime:
ret[instrument] = bars[nextIdx]
self.__nextBarIdx[instrument] += 1
self.__barsLeft -= 1
if self.__prevDateTime == smallestDateTime:
raise Exception("Duplicate bars found for %s on %s" %
(ret.keys(), smallestDateTime))
self.__prevDateTime = smallestDateTime
return bar.Bars(ret)
def testInvalidDateTimes(self):
b1 = bar.BasicBar(datetime.datetime.now(), 2, 3, 1, 2.1, 10, 5,
bar.Frequency.DAY)
b2 = bar.BasicBar(datetime.datetime.now() + datetime.timedelta(days=1),
2, 3, 1, 2.1, 10, 5, bar.Frequency.DAY)
with self.assertRaises(Exception):
bar.Bars({"a": b1, "b": b2})
def buildBars(self, openPrice, highPrice, lowPrice, closePrice, sessionClose = False):
ret = {}
dateTime = self.__getNextDateTime(sessionClose)
bar_ = bar.BasicBar(dateTime, openPrice, highPrice, lowPrice, closePrice, closePrice*10, closePrice)
bar_.setSessionClose(sessionClose)
ret[BaseTestCase.TestInstrument] = bar_
return bar.Bars(ret)
def testDupliateDateTimesForTradeBars(self):
bars = [
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 1, 1, bar.Frequency.TRADE)
]),
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 2, 1, bar.Frequency.TRADE)
]),
]
f = barfeed.OptimizerBarFeed(bar.Frequency.TRADE, [INSTRUMENT], bars)
expected_volume = 1
for dt, b in f:
assert b[INSTRUMENT].getVolume() == expected_volume
expected_volume += 1
def getNextBars(self):
if not self.__data_downloaded:
logger.info('featch history of {0}'.format(self.__instrument))
for i in self.getFrequencies():
tmp = None
if i == Frequency.DAY:
tmp = history(self.__instrument, self.__start_date, None,
Frequency.DAY, add_missing_dates=False)
elif i == Frequency.HOUR:
tmp = history(self.__instrument, self.__start_date, None,
Frequency.HOUR, add_missing_dates=False)
if tmp is None:
continue
for date, row in tmp.iloc[0].iterrows():
tmpbar = bar.BasicBar(date, row['open'], row['high'],
row['low'], row['close'], 0, False,
i)
self.__bars_buf.append(tmpbar)
self.__data_downloaded = True
self.__bars_buf.sort(key=lambda i: i.getDateTime())
while True:
if self.__bars_buf:
tmp = self.__bars_buf.pop(0)
return bar.Bars({self.__instrument: tmp}, frequecy=tmp.getFrequency())
else:
self.generateBars()
def getNextBars(self):
"""Returns the next :class:`pyalgotrade.bar.Bars` in the feed or None if there are no bars."""
barDict = self.fetchNextBars()
if barDict == None:
return None
# TODO: Make this check optional. Default should be NOT to do it.
# Check that bars were retured for all the instruments registered.
# barInstruments = barDict.keys()
# barInstruments.sort()
# registeredInstruments = self.getRegisteredInstruments()
# registeredInstruments.sort()
# if barInstruments != registeredInstruments:
# missing = filter(lambda instrument: instrument not in barInstruments, registeredInstruments)
# raise Exception("Some bars are missing: %s" % missing)
# This will check for incosistent datetimes between bars.
ret = bar.Bars(barDict)
# Check that current bar datetimes are greater than the previous one.
if self.__prevDateTime != None and self.__prevDateTime >= ret.getDateTime():
raise Exception("Bar data times are not in order. Previous datetime was %s and current datetime is %s" % (self.__prevDateTime, ret.getDateTime()))
self.__prevDateTime = ret.getDateTime()
return ret
class TushareBarFeedThread(TuSharePollingThread):
# Events
ON_BARS = 1
def __init__(self, queue, identifiers, frequency):
super(TushareBarFeedThread, self).__init__(identifiers)
self.__queue = queue
self.__frequency = frequency
self.__updateNextBarClose()
def __updateNextBarClose(self):
self.__nextBarClose = resamplebase.build_range(
utcnow(), self.__frequency).getEnding()
def getNextCallDateTime(self):
return self.__nextBarClose
def doCall(self):
endDateTime = self.__nextBarClose
self.__updateNextBarClose()
bar_dict = {}
for identifier in self._identifiers:
try:
if not self._tickDSDict[identifier].empty():
bar_dict[identifier] = build_bar(
to_market_datetime(endDateTime),
self._tickDSDict[identifier])
except Exception, e:
logger.error(e)
if len(bar_dict):
bars = bar.Bars(bar_dict)
self.__queue.put((TushareBarFeedThread.ON_BARS, bars))
def getNextBars(self):
ret = None
try:
ret = bar.Bars(self.__bars.pop(0))
except:
pass
return ret
def doGetHistory(self):
while not self.stopped():
endTimestamp = dt.datetime_to_timestamp(self.__nextBarClose)
self.__updateNextBarClose()
dicts = {}
try:
for indentifier in self.__identifiers:
response = api.getKLineBar(
indentifier, endTimestamp - self.__frequency * 2,
self.__period, 100)
if response is None:
raise Exception("getKLineBar return None!")
dicts[indentifier] = response
break
except:
time.sleep(1)
continue
while not self.stopped():
barDict = {}
for indentifier in self.__identifiers:
response = dicts[indentifier]
if len(response) == 0:
break
barDict[indentifier] = liveBar(response.pop(-1),
self.__frequency)
if len(barDict) == 0:
break
bars = bar.Bars(barDict)
self.__queue.put((GetBarThread.ON_HISTORY_BARS, bars))
def testDupliateDateTimesForDailyBars(self):
bars = [
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 1, 1, bar.Frequency.DAY)
]),
bar.Bars([
bar.BasicBar(INSTRUMENT, datetime.datetime(2001, 1, 1), 1, 1,
1, 1, 1, 1, bar.Frequency.DAY)
]),
]
f = barfeed.OptimizerBarFeed(bar.Frequency.DAY, [INSTRUMENT], bars)
with self.assertRaisesRegexp(Exception, "%s bars are not in order.*" %
INSTRUMENT):
for dt, b in f:
pass
def testInvalidDateTimes(self):
b1 = bar.BasicBar(INSTRUMENT, datetime.datetime.now(), 2, 3, 1, 2.1,
10, 5, bar.Frequency.DAY)
b2 = bar.BasicBar(INSTRUMENT,
datetime.datetime.now() + datetime.timedelta(days=1),
2, 3, 1, 2.1, 10, 5, bar.Frequency.DAY)
with self.assertRaisesRegexp(Exception,
"Bar data times are not in sync"):
bar.Bars([b1, b2])
class GetBarThread(PollingThread):
# Events
ON_BARS = 1
def __init__(self, queue, identifiers, frequency):
super(GetBarThread, self).__init__()
self.__queue = queue
self.__identifiers = identifiers
self.__frequency = frequency
self.__nextBarStart = None
self.__nextBarClose = None
self._ticksDf = {}
self.__lock = Lock()
for identifier in self.__identifiers:
self._ticksDf[identifier] = DataFrame(
columns=['time', 'price', 'volume', 'amount'])
self._ctpMdApi = CTPMdApi(self.__identifiers, self._ticksDf,
self.__lock, logger)
#self._ctpMdApi.login("tcp://222.66.97.241:41213", '9017811', '123456', '7070')
self._ctpMdApi.login("tcp://180.168.212.228:41213", "simnow申请",
"simnow申请", "9999")
#self._ctpMdApi.login("tcp://211.144.195.163:34505", "", "", "")
self.__updateNextBarClose()
def __updateNextBarClose(self):
self.__nextBarStart = resamplebase.build_range(
utcnow(), self.__frequency).getBeginning()
self.__nextBarClose = resamplebase.build_range(
utcnow(), self.__frequency).getEnding()
def getNextCallDateTime(self):
return self.__nextBarClose
def doCall(self):
startDateTime = to_market_datetime(self.__nextBarStart)
endDateTime = to_market_datetime(self.__nextBarClose)
self.__updateNextBarClose()
barDict = dict()
for identifier in self.__identifiers:
try:
period_bar = self._build_bar(identifier, startDateTime,
endDateTime)
if period_bar:
barDict[identifier] = period_bar
except Exception, e:
logger.error(e)
if len(barDict):
bars = bar.Bars(barDict)
self.__queue.put((GetBarThread.ON_BARS, bars))
def getNextBars(self):
ret = {}
for instrument in self.getKeys():
bar = self.__db.getBar(instrument, self.getFrequency(), self.__dateTime)
if bar:
ret[instrument] = bar
self.getNextDateTime()
return bars.Bars(ret)
def getNextBars(self):
df = self._tdx.getQuotes(self.getKeys())
df.set_index(['证券代码'], inplace=True)
dt = datetime.now()
return bar.Bars({code: bar.BasicBar(dt,
float(df['当前价'][code]),
float(df['当前价'][code]),
float(df['当前价'][code]),
float(df['当前价'][code]),
0, None, bar.Frequency.TRADE) for code in df.index})
def __fill_bars(self, bars_dict):
for index, value in enumerate(bars_dict[self.__identifiers[0]]):
bar_dict = dict()
for identifier in self.__identifiers:
if bars_dict[identifier][index] is not None:
bar_dict[identifier] = bars_dict[identifier][index]
if len(bar_dict):
bars = bar.Bars(bar_dict)
self.__queue.put((TushareBarFeedThread.ON_BARS, bars))
def testIter(self):
dt = datetime.datetime.now()
b1 = bar.BasicBar("a/USD", dt, 1, 1, 1, 1, 10, 1, bar.Frequency.DAY)
b2 = bar.BasicBar("a/EUR", dt, 2, 2, 2, 2, 10, 2, bar.Frequency.DAY)
b3 = bar.BasicBar("b/USD", dt, 2, 2, 2, 2, 10, 2, bar.Frequency.DAY)
bars = bar.Bars([b1, b2, b3])
items = [b1, b2, b3]
for item in bars:
items.remove(item)
assert len(items) == 0
def testBasic(self):
dt = datetime.datetime.now()
b1 = bar.BasicBar(dt, 1, 1, 1, 1, 10, 1, bar.Frequency.DAY)
b2 = bar.BasicBar(dt, 2, 2, 2, 2, 10, 2, bar.Frequency.DAY)
bars = bar.Bars({"a": b1, "b": b2})
self.assertEqual(bars["a"].getClose(), 1)
self.assertEqual(bars["b"].getClose(), 2)
self.assertTrue("a" in bars)
self.assertEqual(bars.items(), [("a", b1), ("b", b2)])
self.assertEqual(bars.keys(), ["a", "b"])
self.assertEqual(bars.getInstruments(), ["a", "b"])
self.assertEqual(bars.getDateTime(), dt)
self.assertEqual(bars.getBar("a").getClose(), 1)
def getNextBars(self):
self.getNextDateTime()
self.getNextMembers()
ret = self.__db.getBars(self.__instruments, self.getFrequency(), self.__dateTime)
for i in range(len(self.__instruments) - 1, -1, -1):
instrument = self.__instruments[i]
if instrument not in ret:
self.__instruments.pop(i)
self.getNextDatePos()
return bars.Bars(ret)
def doCall(self):
barDict = {}
self.__updateNextBarClose()
for indentifier in self.__identifiers:
try:
response = ts.get_k_data(code=indentifier, ktype=self.__precision)
barDict[indentifier] = build_bar(response.iloc[-1], self.__frequency)
except:
logger.error("tushare time out")
if len(barDict):
bars = bar.Bars(barDict)
self.__queue.put((GetBarThread.ON_BARS, bars))
def testBasicOperations(self):
dt = datetime.datetime.now()
b1 = bar.BasicBar("a/usd", dt, 1, 1, 1, 1, 10, 1, bar.Frequency.DAY)
b2 = bar.BasicBar("b/usd", dt, 2, 2, 2, 2, 10, 2, bar.Frequency.DAY)
bars = bar.Bars([b1, b2])
self.assertEqual(bars["a/usd"].getClose(), 1)
self.assertEqual(bars["b/usd"].getClose(), 2)
self.assertTrue("a/usd" in bars)
self.assertEqual(bars.getInstruments(), ["a/usd", "b/usd"])
self.assertEqual(bars.getDateTime(), dt)
self.assertEqual(bars.getBar("a/usd").getClose(), 1)
with self.assertRaises(KeyError):
bars["c/usd"]
def doCall(self):
endDateTime = self.__nextBarClose
self.__updateNextBarClose()
barDict = {}
for indentifier in self.__identifiers:
try:
logger.debug("Requesting bars with precision %s and period %s for %s" % (self.__precision, self.__period, indentifier))
response = api.XigniteGlobalRealTime_GetBar(self.__apiToken, indentifier, "Symbol", endDateTime, self.__precision, self.__period)
# logger.debug(response)
barDict[indentifier] = build_bar(response["Bar"], indentifier, self.__frequency)
except api.XigniteError as e:
logger.error(e)
if len(barDict):
bars = bar.Bars(barDict)
self.__queue.put((GetBarThread.ON_BARS, bars))
def getNextBars(self):
# All bars must have the same datetime. We will return all the ones with the smallest datetime.
smallestDateTime = self.peekDateTime()
if smallestDateTime == None:
assert(self.__barsLeft == 0)
return None
# Make a second pass to get all the bars that had the smallest datetime.
ret = {}
for instrument, bars in self.__bars.iteritems():
nextIdx = self.__nextBarIdx[instrument]
if nextIdx < len(bars) and bars[nextIdx].getDateTime() == smallestDateTime:
ret[instrument] = bars[nextIdx]
self.__nextBarIdx[instrument] += 1
self.__barsLeft -= 1
return bar.Bars(ret)
def getNextBars(self):
"""Returns the next :class:`pyalgotrade.bar.Bars` in the feed or None if there are no bars."""
barDict = self.fetchNextBars()
if barDict == None:
return None
# This will check for incosistent datetimes between bars.
ret = bar.Bars(barDict)
# Check that current bar datetimes are greater than the previous one.
if self.__prevDateTime != None and self.__prevDateTime >= ret.getDateTime(
):
raise Exception(
"Bar date times are not in order. Previous datetime was %s and current datetime is %s"
% (self.__prevDateTime, ret.getDateTime()))
self.__prevDateTime = ret.getDateTime()
return ret
def getNextBars(self):
# All bars must have the same datetime. We will return all the ones with the smallest datetime.
smallestDateTime = self.peekDateTime()
if smallestDateTime is None:
return None
# Make a second pass to get all the bars that had the smallest datetime.
ret = {}
for instrument, bars in six.iteritems(self.__bars):
nextPos = self.__nextPos[instrument]
if nextPos < len(bars) and bars[nextPos].getDateTime() == smallestDateTime:
ret[instrument] = bars[nextPos]
self.__nextPos[instrument] += 1
if self.__currDateTime == smallestDateTime:
raise Exception("Duplicate bars found for %s on %s" % (list(ret.keys()), smallestDateTime))
self.__currDateTime = smallestDateTime
return bar.Bars(ret)
def doCall(self):
endTimestamp = dt.datetime_to_timestamp(self.__nextBarClose)
barDict = {}
for indentifier in self.__identifiers:
try:
response = api.getKLineBar(indentifier, endTimestamp - self.__frequency, self.__period)
if response is None:
raise Exception("getKLineBar return None!")
# logger.debug(response)
barDict[indentifier] = liveBar(response[-1], self.__frequency)
except:
time.sleep(1)
return False
if len(barDict):
bars = bar.Bars(barDict)
self.__queue.put((GetBarThread.ON_BARS, bars))
self.__updateNextBarClose()
return True
return False
def doCall(self):
barDict = {}
self.__updateNextBarClose()
index_time = ts.get_k_data(code='000001',
ktype=self.__precision,
index=True).date.values[-1]
for identifier in self.__identifiers:
try:
logger.info('loading data online: {0}'.format(identifier))
response = ts.get_k_data(code=identifier,
ktype=self.__precision)
response_time = response.date.values[-1]
if response_time >= index_time:
barDict[identifier] = build_bar(
response[response.date == index_time].to_dict(),
self.__frequency) #以指数时间为基准，保持bar的一致性
except:
logger.error("tushare time out")
if len(barDict):
bars = bar.Bars(barDict)
self.__queue.put((GetBarThread.ON_BARS, bars))
def getNextBars(self):
# All bars must have the same datetime. We will return all the ones with the smallest datetime.
smallestDateTime = self.peekDateTime()
if smallestDateTime is None:
return None
# Make a second pass to get all the bars that have the smallest datetime.
ret = []
for instrument, bars in six.iteritems(self.__bars):
nextPos = self.__nextPos[instrument]
if nextPos < len(bars) and bars[nextPos].getDateTime(
) == smallestDateTime:
# Check if there are duplicate bars (with the same datetime).
if self.__currDateTime == smallestDateTime:
raise Exception("Duplicate bars found for %s on %s" %
(instrument, smallestDateTime))
assert bars[nextPos].getInstrument() == instrument, \
"%s != %s" % (bars[nextPos].getInstrument(), instrument)
ret.append(bars[nextPos])
self.__nextPos[instrument] += 1
self.__currDateTime = smallestDateTime
return bar.Bars(ret)
