def loadBar(self, dbName, collectionName, days):
"""从数据库中读取Bar数据，startDate是datetime对象"""
startDate = self.today - timedelta(days)
d = {'datetime':{'$gte':startDate}}
barData = self.mainEngine.dbQuery(dbName, collectionName, d)
l = []
for d in barData:
bar = VtBarData()
bar.__dict__ = d
l.append(bar)
return l
def loadBar(self, dbName, collectionName, days):
"""从数据库中读取Bar数据，startDate是datetime对象"""
print "%s.%s.%s" % (__name__, self.__class__.__name__, get_current_function_name())
startDate = self.today - timedelta(days)
d = {'datetime': {'$gte': startDate}}
barData = self.mainEngine.dbQuery(dbName, collectionName, d, 'datetime')
l = []
for d in barData:
bar = VtBarData()
bar.__dict__ = d
l.append(bar)
return l
def loadBar(self, dbName, collectionName, days):
"""从数据库中读取Bar数据，startDate是datetime对象"""
# 优先尝试从RQData获取数据
if dbName == MINUTE_DB_NAME and collectionName.upper() in self.rqSymbolSet:
l = self.loadRqBar(collectionName, days)
return l
# 如果没有则从数据库中读取数据
startDate = self.today - timedelta(days)
d = {'datetime':{'$gte':startDate}}
barData = self.mainEngine.dbQuery(dbName, collectionName, d, 'datetime')
l = []
for d in barData:
bar = VtBarData()
bar.__dict__ = d
l.append(bar)
return l
def loadData(self):
"""加载数据"""
mc = MongoClient()
db = mc[DAILY_DB_NAME]
for vtSymbol in self.vtSymbolList:
flt = {'datetime': {'$gte': self.startDt, '$lte': self.endDt}}
collection = db[vtSymbol]
cursor = collection.find(flt).sort('datetime')
for d in cursor:
bar = VtBarData()
bar.__dict__ = d
barDict = self.dataDict.setdefault(bar.datetime, OrderedDict())
barDict[bar.vtSymbol] = bar
self.output(u'%s数据加载完成，总数据量：%s' % (vtSymbol, cursor.count()))
self.output(u'全部数据加载完成')
def loadBar(self, dbName, collectionName, days):
"""从数据库中读取Bar数据，startDate是datetime对象"""
# 优先尝试从RQData获取数据
if dbName == MINUTE_DB_NAME and collectionName.upper(
) in self.rqSymbolSet:
l = self.loadRqBar(collectionName, days)
return l
# 如果没有则从数据库中读取数据
startDate = self.today - timedelta(days)
d = {'datetime': {'$gte': startDate}}
barData = self.mainEngine.dbQuery(dbName, collectionName, d,
'datetime')
l = []
for d in barData:
bar = VtBarData()
bar.__dict__ = d
l.append(bar)
return l
def new_bar_from_tick(self, tick, freq=None):
bar = VtBarData()
bar.vtSymbol = tick.vtSymbol
bar.symbol = tick.symbol
bar.exchange = tick.exchange
bar.gatewayName = tick.gatewayName
return self.override_bar_with_tick(bar, tick, freq=freq)
def updateDay(self, bar):
newDay = False
# 尚未创建对象
if not self.dayBar:
self.dayBar = VtBarData()
newDay = True
elif self.dayBar.TradingDay != bar.TradingDay:
print "updateDay:%s" % bar.date
# 生成上一X分钟K线的时间戳
self.dayBar.datetime = self.dayBar.datetime.replace(hour=0,minute=0,second=0, microsecond=0) # 将秒和微秒设为0
self.dayBar.date = self.dayBar.datetime.strftime('%Y%m%d')
self.dayBar.time = self.dayBar.datetime.strftime('%H:%M:%S.%f')
# 推送
self.onDayBar(self.dayBar)
# 清空老K线缓存对象
self.dayBar = VtBarData()
newDay = True
if newDay:
self.dayBar.vtSymbol = bar.vtSymbol
self.dayBar.symbol = bar.symbol
self.dayBar.exchange = bar.exchange
self.dayBar.TradingDay = bar.TradingDay
self.dayBar.open = bar.open
self.dayBar.high = bar.high
self.dayBar.low = bar.low
else:
# 累加老K线
self.dayBar.high = max(self.dayBar.high, bar.high)
self.dayBar.low = min(self.dayBar.low, bar.low)
# 通用部分
self.dayBar.close = bar.close
self.dayBar.datetime = bar.datetime
self.dayBar.openInterest = bar.openInterest
self.dayBar.volume += int(bar.volume)
def addSymbolFreq(self, symbol, freq, size = 50):
key = symbol + '_' + freq
self.symbolFreqData[key] = ArrayManager(size=size)
data_list=[]
if freq == '1MIN':
data_list = self.mainEngine.dbQuery( MINUTE_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
elif freq == '5MIN':
data_list = self.mainEngine.dbQuery( MINUTE_5_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
elif freq == '15MIN':
data_list = self.mainEngine.dbQuery( MINUTE_15_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
elif freq == '30MIN':
data_list = self.mainEngine.dbQuery( MINUTE_30_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
elif freq == '60MIN':
data_list = self.mainEngine.dbQuery( MINUTE_60_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
elif freq == 'D':
data_list = self.mainEngine.dbQuery( DAILY_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
elif freq == 'W':
data_list = self.mainEngine.dbQuery( WEEKLY_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
if np.size(data_list) >= size:
data_list = data_list[-1*size:]
for data in data_list:
bar = VtBarData()
bar.close = data['close']
bar.open = data['open']
bar.high = data['high']
bar.low = data['low']
self.symbolFreqData[key].updateBar(bar)
def new_bar_from_bar(self, bar, freq=None):
bar2 = VtBarData()
bar2.vtSymbol = bar.vtSymbol
bar2.symbol = bar.symbol
bar2.exchange = bar.exchange
bar2.gatewayName = bar.gatewayName
return self.override_bar_with_bar(bar2, bar, freq=freq)
def loadData(self):
"""加载数据"""
mc = MongoClient()
db = mc[DAILY_DB_NAME]
for vtSymbol in self.vtSymbolList:
flt = {'datetime':{'$gte':self.startDt,
'$lte':self.endDt}}
collection = db[vtSymbol]
cursor = collection.find(flt).sort('datetime')
for d in cursor:
bar = VtBarData()
bar.__dict__ = d
barDict = self.dataDict.setdefault(bar.datetime, OrderedDict())
barDict[bar.vtSymbol] = bar
self.output(u'%s数据加载完成，总数据量：%s' %(vtSymbol, cursor.count()))
self.output(u'全部数据加载完成')
def onTick(self, tick):
"""收到行情TICK推送（必须由用户继承实现）"""
bar = VtBarData()
bar.open = tick.lastPrice
bar.close = tick.lastPrice
bar.low = tick.lastPrice
bar.high = tick.lastPrice
self.onBar(bar)
def loadCsv(filename):
""""""
symbol = filename.split('.')[0]
mc = MongoClient()
db = mc[DAILY_DB_NAME]
collection = db[symbol]
with open(filename) as f:
r = DictReader(f)
for d in r:
bar = VtBarData()
bar.datetime = datetime.strptime(d['date'], '%Y/%m/%d')
bar.vtSymbol = symbol
bar.open = float(d['open'])
bar.high = float(d['high'])
bar.low = float(d['low'])
bar.close = float(d['close'])
bar.volume = int(d['volume'])
collection.insert(bar.__dict__)
def updateWCandle(self, Candle):
"""周K线更新"""
# 尚未创建对象
abstract_week = Candle.datetime.strftime('%W')
if not self.intraWeek:
self.intraWeek = abstract_week
if abstract_week != self.intraWeek:
# 推送
if self.WeekCandle:
self.onWCandle(self.WeekCandle)
# 清空老K线缓存对象
self.WeekCandle = None
if not self.WeekCandle:
self.WeekCandle = VtBarData()
self.WeekCandle.vtSymbol = Candle.vtSymbol
self.WeekCandle.symbol = Candle.symbol
self.WeekCandle.exchange = Candle.exchange
self.WeekCandle.open = Candle.open
self.WeekCandle.high = Candle.high
self.WeekCandle.low = Candle.low
self.WeekCandle.datetime = Candle.datetime.replace(
second=0, microsecond=0) # 将秒和微秒设为0
self.WeekCandle.date = Candle.datetime.strftime('%Y%m%d')
self.WeekCandle.time = Candle.datetime.strftime('%H:%M:%S.%f')
# 累加老K线
self.WeekCandle.high = max(self.WeekCandle.high, Candle.high)
self.WeekCandle.low = min(self.WeekCandle.low, Candle.low)
self.WeekCandle.close = Candle.close
self.WeekCandle.openInterest = Candle.openInterest
self.WeekCandle.volume += Candle.volume
self.intraWeek = abstract_week
if (bar.datetime.hour, bar.datetime.minute
) == self.marketClose and self.marketClose != (23, 59):
if Candle.datetime.strftime('%w') == 5: # 每周五收盘强切周线
self.onWCandle(self.WeekCandle)
self.WeekCandle = None
elif (bar.datetime.hour, bar.datetime.minute
) == self.marketClose and self.marketClose == (23, 59):
if Candle.datetime.strftime('%w') == 0: # 7*24市场在周日晚0点切
self.onWCandle(self.WeekCandle)
self.WeekCandle = None
def updateBar(self, bar):
"""1分钟K线更新"""
# 尚未创建对象
if not self.xminBar:
self.xminBar = VtBarData()
self.xminBar.vtSymbol = bar.vtSymbol
self.xminBar.symbol = bar.symbol
self.xminBar.exchange = bar.exchange
self.xminBar.open = bar.open
self.xminBar.high = bar.high
self.xminBar.low = bar.low
self.xminBar.datetime = bar.datetime # 以第一根分钟K线的开始时间戳作为X分钟线的时间戳
# 累加老K线
else:
self.xminBar.high = max(self.xminBar.high, bar.high)
self.xminBar.low = min(self.xminBar.low, bar.low)
# 通用部分
self.xminBar.close = bar.close
self.xminBar.openInterest = bar.openInterest
self.xminBar.volume += int(bar.volume)
# X分钟已经走完
if not (bar.datetime.minute + 1) % self.xmin: # 可以用X整除
# 生成上一X分钟K线的时间戳
self.xminBar.datetime = self.xminBar.datetime.replace(
second=0, microsecond=0) # 将秒和微秒设为0
self.xminBar.date = self.xminBar.datetime.strftime('%Y%m%d')
self.xminBar.time = self.xminBar.datetime.strftime('%H:%M:%S.%f')
# 推送
if self.xminBar.low == 0 or self.xminBar.high == 0 or self.xminBar.low == self.xminBar.high:
return
else:
self.onXminBar(self.xminBar, self.lastTick)
# 清空老K线缓存对象
self.xminBar = None
# 更新交易日
self.tradingDay
def loadCsv(filename):
""""""
symbol = filename.split('.')[0]
mc = MongoClient()
db = mc[DAILY_DB_NAME]
collection = db[symbol]
with open(filename) as f:
r = DictReader(f)
for d in r:
bar = VtBarData()
bar.datetime = datetime.strptime(d['date'], '%Y/%m/%d')
bar.vtSymbol = symbol
bar.open = float(d['open'])
bar.high = float(d['high'])
bar.low = float(d['low'])
bar.close = float(d['close'])
bar.volume= int(d['volume'])
collection.insert(bar.__dict__)
def updateBar(self, bar):
"""1分钟K线更新"""
# 尚未创建对象
if not self.xminBar:
self.xminBar = VtBarData()
self.xminBar.vtSymbol = bar.vtSymbol
self.xminBar.symbol = bar.symbol
self.xminBar.exchange = bar.exchange
self.xminBar.open = bar.open
self.xminBar.high = bar.high
self.xminBar.low = bar.low
# 生成上一X分钟K线的时间戳
self.xminBar.datetime = bar.datetime.replace(
second=0, microsecond=0) # 将秒和微秒设为0
self.xminBar.date = bar.datetime.strftime('%Y%m%d')
self.xminBar.time = bar.datetime.strftime('%H:%M:%S.%f')
# 累加老K线
else:
self.xminBar.high = max(self.xminBar.high, bar.high)
self.xminBar.low = min(self.xminBar.low, bar.low)
# 通用部分
self.xminBar.close = bar.close
# self.xminBar.datetime = bar.datetime
self.xminBar.openInterest = bar.openInterest
self.xminBar.volume += int(bar.volume)
# X分钟已经走完
if self.xmin < 61:
if not (bar.datetime.minute + 1) % self.xmin: # 可以用X整除
# 推送
self.onXminBar(self.xminBar)
# 清空老K线缓存对象
self.xminBar = None
elif self.xmin > 60:
if not (bar.datetime.hour *
60) % self.xmin and bar.datetime.minute == 0: # 小时线
self.onXminBar(self.xminBar)
# 清空老K线缓存对象
self.xminBar = None
def updateBar(self, bar):
"""1分钟K线更新"""
# 尚未创建对象
if not self.xminBar:
self.xminBar = VtBarData()
self.xminBar.vtSymbol = bar.vtSymbol
self.xminBar.symbol = bar.symbol
self.xminBar.exchange = bar.exchange
self.xminBar.open = bar.open
self.xminBar.high = bar.high
self.xminBar.low = bar.low
# 累加老K线
else:
self.xminBar.high = max(self.xminBar.high, bar.high)
self.xminBar.low = min(self.xminBar.low, bar.low)
# 通用部分
self.xminBar.close = bar.close
self.xminBar.datetime = bar.datetime
self.xminBar.openInterest = bar.openInterest
self.xminBar.volume += int(bar.volume)
# X分钟已经走完
# if not bar.datetime.minute % self.xmin: # 可以用X整除
if not (bar.datetime.minute +
1) % self.xmin: # 可以用X整除，原写法会造成早上开盘第一根bar就算合成了一根bar，张英杰修改
# 生成上一X分钟K线的时间戳
self.xminBar.datetime = self.xminBar.datetime.replace(
second=0, microsecond=0) # 将秒和微秒设为0
self.xminBar.date = self.xminBar.datetime.strftime('%Y%m%d')
self.xminBar.time = self.xminBar.datetime.strftime('%H:%M:%S.%f')
# 推送
self.onXminBar(self.xminBar)
# 清空老K线缓存对象
self.xminBar = None
def updateMyBar(self, minute, bar):
"""1分钟K线更新"""
# 尚未创建对象
if not self.myXminBar[minute]:
self.myXminBar[minute] = VtBarData()
self.myXminBar[minute].vtSymbol = bar.vtSymbol
self.myXminBar[minute].symbol = bar.symbol
self.myXminBar[minute].exchange = bar.exchange
self.myXminBar[minute].open = bar.open
self.myXminBar[minute].high = bar.high
self.myXminBar[minute].low = bar.low
# 累加老K线
else:
self.myXminBar[minute].high = max(self.myXminBar[minute].high,
bar.high)
self.myXminBar[minute].low = min(self.myXminBar[minute].low,
bar.low)
# 通用部分
self.myXminBar[minute].close = bar.close
self.myXminBar[minute].datetime = bar.datetime
self.myXminBar[minute].openInterest = bar.openInterest
self.myXminBar[minute].volume += int(bar.volume)
# X分钟已经走完
if (not bar.datetime.minute % minute): # 可以用X整除
# 生成上一X分钟K线的时间戳
self.myXminBar[minute].datetime = self.myXminBar[
minute].datetime.replace(second=0, microsecond=0) # 将秒和微秒设为0
self.myXminBar[minute].date = self.myXminBar[
minute].datetime.strftime('%Y%m%d')
self.myXminBar[minute].time = self.myXminBar[
minute].datetime.strftime('%H:%M:%S.%f')
# 推送
self.onXBar(minute, self.myXminBar[minute])
# 清空老K线缓存对象
self.myXminBar[minute] = None
def updateDay(self, bar):
"""天更新"""
# 尚未创建对象
if not self.dayBar:
self.dayBar = VtBarData()
self.dayBar.vtSymbol = bar.vtSymbol
self.dayBar.symbol = bar.symbol
self.dayBar.exchange = bar.exchange
self.dayBar.open = bar.open
self.dayBar.high = bar.high
self.dayBar.low = bar.low
# 累加老K线
else:
self.dayBar.high = max(self.dayBar.high, bar.high)
self.dayBar.low = min(self.dayBar.low, bar.low)
# 通用部分
self.dayBar.close = bar.close
self.dayBar.datetime = bar.datetime
self.dayBar.openInterest = bar.openInterest
self.dayBar.volume += int(bar.volume)
# 判断天数
if bar.datetime.time() == self.DAY_END: #
print "updateDay:%s" % bar.date
# 生成上一X分钟K线的时间戳
self.dayBar.datetime = self.dayBar.datetime.replace(
hour=0, minute=0, second=0, microsecond=0) # 将秒和微秒设为0
self.dayBar.date = self.dayBar.datetime.strftime('%Y%m%d')
self.dayBar.time = self.dayBar.datetime.strftime('%H:%M:%S.%f')
# 推送
self.onDayBar(self.dayBar)
# 清空老K线缓存对象
self.dayBar = None
def updateGivingBar(self, bar):
#givingBar这个变量只在本函数内使用
if not self.givingBar:
self.givingBar = VtBarData()
self.givingBar.vtSymbol = bar.vtSymbol
self.givingBar.symbol = bar.symbol
self.givingBar.exchange = bar.exchange
self.givingBar.TradingDay = bar.TradingDay
self.givingBar.open = bar.open
self.givingBar.high = bar.high
self.givingBar.low = bar.low
self.givingBar.datetime = bar.datetime
else:
self.givingBar.high = max(self.givingBar.high, bar.high)
self.givingBar.low = min(self.givingBar.low, bar.low)
# 通用部分
self.givingBar.close = bar.close
self.givingBar.openInterest = bar.openInterest
self.givingBar.volume += int(bar.volume)
def __firstTick(self, tick):
""" K线的第一个Tick数据"""
self.bar = VtBarData() # 创建新的K线
self.bar.vtSymbol = tick.vtSymbol
self.bar.symbol = tick.symbol
self.bar.exchange = tick.exchange
self.bar.open = tick.lastPrice # O L H C
self.bar.high = tick.lastPrice
self.bar.low = tick.lastPrice
self.bar.close = tick.lastPrice
# K线的日期时间
self.bar.date = tick.date # K线的日期时间（去除秒）设为第一个Tick的时间
self.bar.time = tick.time # K线的日期时间（去除秒）设为第一个Tick的时间
self.bar.datetime = tick.datetime
self.bar.volume = tick.volume
self.bar.openInterest = tick.openInterest
self.barFirstTick = True # 标识该Tick属于该Bar的第一个tick数据
self.lineBar.append(self.bar) # 推入到lineBar队列
def updateMCandle(self, Candle):
"""月K线更新"""
# 尚未创建对象
abstract_month = int(Candle.datetime.strftime('%m'))
if not self.intraMonth:
self.intraMonth = abstract_month
if abstract_month != self.intraMonth:
# 推送
if self.MonthCandle:
self.onMCandle(self.MonthCandle)
# 清空老K线缓存对象
self.MonthCandle = None
if not self.MonthCandle:
self.MonthCandle = VtBarData()
self.MonthCandle.vtSymbol = Candle.vtSymbol
self.MonthCandle.symbol = Candle.symbol
self.MonthCandle.exchange = Candle.exchange
self.MonthCandle.open = Candle.open
self.MonthCandle.high = Candle.high
self.MonthCandle.low = Candle.low
self.MonthCandle.datetime = Candle.datetime.replace(
second=0, microsecond=0) # 将秒和微秒设为0
self.MonthCandle.date = Candle.datetime.strftime('%Y%m%d')
self.MonthCandle.time = Candle.datetime.strftime('%H:%M:%S.%f')
# 累加老K线
self.MonthCandle.high = max(self.MonthCandle.high, Candle.high)
self.MonthCandle.low = min(self.MonthCandle.low, Candle.low)
self.MonthCandle.close = Candle.close
self.MonthCandle.openInterest = Candle.openInterest
self.MonthCandle.volume += Candle.volume
self.intraMonth = abstract_month
def loadTdxCsv(fileName, dbName, symbol):
"""将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
date_correct = ""
print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[1])
bar.high = float(d[2])
bar.low = float(d[3])
bar.close = float(d[4])
#通达信的夜盘时间按照新的一天计算，此处将其按照当天日期统计，方便后续查阅
date_temp, time_temp = d[0].strip(' ').replace('\xef\xbb\xbf',
'').split('-', 1)
if time_temp == '15:00':
date_correct = date_temp
if time_temp[:
2] == "21" or time_temp[:
2] == "22" or time_temp[:
2] == "23":
date_temp = date_correct
bar.date = datetime.strptime(date_temp, '%Y/%m/%d').strftime('%Y%m%d')
bar.time = time_temp[:2] + ':' + time_temp[3:5] + ':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d[5]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print('插入完毕，耗时：%s' % (time() - start))
def generateVtBar(vtSymbol, d):
"""生成K线"""
bar = VtBarData()
bar.vtSymbol = vtSymbol
bar.symbol, bar.exchange = bar.vtSymbol.split('.')
bar.datetime = datetime.datetime.fromtimestamp(d['time'])
bar.date = bar.datetime.strftime('%Y%m%d')
bar.time = bar.datetime.strftime('%H:%M:%S')
bar.open = d['open']
bar.high = d['high']
bar.low = d['low']
bar.close = d['close']
bar.volume = d['volumeto']
return bar
def generateVtBar(symbol, d):
"""生成K线"""
bar = VtBarData()
bar.symbol = symbol
bar.vtSymbol = symbol
bar.open = d['open']
bar.high = d['high']
bar.low = d['low']
bar.close = d['close']
bar.volume = d['volume']
bar.openInterest = d['open_oi']
bar.datetime = datetime.fromtimestamp(d['datetime'] / 1000000000)
bar.date = bar.datetime.strftime("%Y%m%d")
bar.time = bar.datetime.strftime("%H:%M:%S")
return bar
def downloadFuturesDailyBar(self, symbol):
"""
下载期货合约的日行情，symbol是合约代码，
若最后四位为0000（如IF0000），代表下载连续合约。
"""
print u'开始下载%s日行情' % symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 主力合约
if '0000' in symbol:
path = 'api/market/getMktMFutd.json'
params = {}
params['contractObject'] = symbol.replace('0000', '')
params['mainCon'] = 1
if last:
params['startDate'] = last['date']
# 交易合约
else:
path = 'api/market/getMktFutd.json'
params = {}
params['ticker'] = symbol
if last:
params['startDate'] = last['date']
# 开始下载数据
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
bar.openInterest = d.get('openInt', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
def downloadEquityDailyBar(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print u'开始下载%s日行情' % symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
path = 'api/market/getMktEqud.json'
params = {}
params['ticker'] = symbol
if last:
params['beginDate'] = last['date']
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
def generateVtBar(symbol, d):
"""生成K线"""
l = symbol.split('_')
bar = VtBarData()
bar.symbol = l[-2] + l[-1]
bar.exchange = l[0]
bar.vtSymbol = '/'.join([bar.symbol, bar.exchange])
bar.datetime = datetime.datetime.strptime(d['time_open'], '%Y-%m-%dT%H:%M:%S.%f0Z')
bar.date = bar.datetime.strftime('%Y%m%d')
bar.time = bar.datetime.strftime('%H:%M:%S')
bar.open = d['price_open']
bar.high = d['price_high']
bar.low = d['price_low']
bar.close = d['price_close']
bar.volume = d['volume_traded']
return bar
def loadTdxLc1(fileName, dbName, symbol):
"""将通达信导出的lc1格式的历史分钟数据插入到Mongo数据库中"""
from struct import *
start = time()
print u'开始读取通达信Lc1文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
#读取二进制文件
ofile=open(fileName,'rb')
buf=ofile.read()
ofile.close()
num=len(buf)
no=num/32
b=0
e=32
dl = []
for i in xrange(no):
a=unpack('hhfffffii',buf[b:e])
b=b+32
e=e+32
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = a[2]
bar.high = a[3]
bar.low = a[4]
bar.close = a[5]
bar.date = str(int(a[0]/2048)+2004)+str(int(a[0]%2048/100)).zfill(2)+str(a[0]%2048%100).zfill(2)
bar.time = str(int(a[1]/60)).zfill(2)+':'+str(a[1]%60).zfill(2)+':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = a[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'插入完毕，耗时：%s' % (time()-start)
def loadTdxCsv(fileName, dbName, symbol):
"""将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.date = datetime.strptime(d[0], '%Y/%m/%d').strftime('%Y%m%d')
bar.time = d[1][:2]+':'+d[1][2:4]+':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = d[6]
bar.openInterest = d[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print bar.date, bar.time
print u'插入完毕，耗时：%s' % (time()-start)
def downloadFuturesDailyBar(self, symbol):
"""
下载期货合约的日行情，symbol是合约代码，
若最后四位为0000（如IF0000），代表下载连续合约。
"""
print u'开始下载%s日行情' %symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 主力合约
if '0000' in symbol:
path = 'api/market/getMktMFutd.json'
params = {}
params['contractObject'] = symbol.replace('0000', '')
params['mainCon'] = 1
if last:
params['startDate'] = last['date']
# 交易合约
else:
path = 'api/market/getMktFutd.json'
params = {}
params['ticker'] = symbol
if last:
params['startDate'] = last['date']
# 开始下载数据
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
bar.openInterest = d.get('openInt', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
def onBar(self, bar):
"""收到Bar推送（必须由用户继承实现）"""
# 如果当前是一个5分钟走完
if bar.datetime.minute % 5 == 0:
# 如果已经有聚合5分钟K线
if self.min5Bar:
# 将最新分钟的数据更新到目前5分钟线中
min5Bar = self.min5Bar
min5Bar.high = max(min5Bar.high, bar.high)
min5Bar.low = min(min5Bar.low, bar.low)
min5Bar.close = bar.close
# 推送5分钟线数据
self.onFiveBar(min5Bar)
# 清空5分钟线数据缓存
self.min5Bar = None
else:
# 如果没有缓存则新建
if not self.min5Bar:
min5Bar = VtBarData()
min5Bar.vtSymbol = bar.vtSymbol
min5Bar.symbol = bar.symbol
min5Bar.exchange = bar.exchange
min5Bar.open = bar.open
min5Bar.high = bar.high
min5Bar.low = bar.low
min5Bar.close = bar.close
min5Bar.date = bar.date
min5Bar.time = bar.time
min5Bar.datetime = bar.datetime
self.min5Bar = min5Bar
else:
min5Bar = self.min5Bar
min5Bar.high = max(min5Bar.high, bar.high)
min5Bar.low = min(min5Bar.low, bar.low)
min5Bar.close = bar.close
# ----------------------------------------------------------------------
if bar.datetime.minute % 15 == 0:
# 如果已经有聚合15分钟K线
if self.min15Bar:
# 将最新分钟的数据更新到目前5分钟线中
min15Bar = self.min15Bar
min15Bar.high = max(min15Bar.high, bar.high)
min15Bar.low = min(min15Bar.low, bar.low)
min15Bar.close = bar.close
# 推送5分钟线数据
self.onFifteenBar(min15Bar)
# 清空5分钟线数据缓存
self.min15Bar = None
else:
# 如果没有缓存则新建
if not self.min15Bar:
min15Bar = VtBarData()
min15Bar.vtSymbol = bar.vtSymbol
min15Bar.symbol = bar.symbol
min15Bar.exchange = bar.exchange
min15Bar.open = bar.open
min15Bar.high = bar.high
min15Bar.low = bar.low
min15Bar.close = bar.close
min15Bar.date = bar.date
min15Bar.time = bar.time
min15Bar.datetime = bar.datetime
self.min15Bar = min15Bar
else:
min15Bar = self.min15Bar
min15Bar.high = max(min15Bar.high, bar.high)
min15Bar.low = min(min15Bar.low, bar.low)
min15Bar.close = bar.close
# ----------------------------------------------------------------------
# 如果当前是一个30分钟走完
if bar.datetime.minute % 30 == 0:
# 如果已经有聚合30分钟K线
if self.min30Bar:
# 将最新分钟的数据更新到目前5分钟线中
min30Bar = self.min30Bar
min30Bar.high = max(min30Bar.high, bar.high)
min30Bar.low = min(min30Bar.low, bar.low)
min30Bar.close = bar.close
# 推送5分钟线数据
self.onThirtyBar(min30Bar)
# 清空5分钟线数据缓存
self.min30Bar = None
else:
# 如果没有缓存则新建
if not self.min30Bar:
min30Bar = VtBarData()
min30Bar.vtSymbol = bar.vtSymbol
min30Bar.symbol = bar.symbol
min30Bar.exchange = bar.exchange
min30Bar.open = bar.open
min30Bar.high = bar.high
min30Bar.low = bar.low
min30Bar.close = bar.close
min30Bar.date = bar.date
min30Bar.time = bar.time
min30Bar.datetime = bar.datetime
self.min30Bar = min30Bar
else:
min30Bar = self.min30Bar
min30Bar.high = max(min30Bar.high, bar.high)
min30Bar.low = min(min30Bar.low, bar.low)
min30Bar.close = bar.close
# ----------------------------------------------------------------------
# 如果当前是一个60分钟走完
if bar.datetime.minute % 60 == 0:
# 如果已经有聚合60分钟K线
if self.min60Bar:
# 将最新分钟的数据更新到目前5分钟线中
min60Bar = self.min60Bar
min60Bar.high = max(min60Bar.high, bar.high)
min60Bar.low = min(min60Bar.low, bar.low)
min60Bar.close = bar.close
# 推送5分钟线数据
self.onSixtyBar(min60Bar)
# 清空5分钟线数据缓存
self.min60Bar = None
else:
# 如果没有缓存则新建
if not self.min60Bar:
min60Bar = VtBarData()
min60Bar.vtSymbol = bar.vtSymbol
min60Bar.symbol = bar.symbol
min60Bar.exchange = bar.exchange
min60Bar.open = bar.open
min60Bar.high = bar.high
min60Bar.low = bar.low
min60Bar.close = bar.close
min60Bar.date = bar.date
min60Bar.time = bar.time
min60Bar.datetime = bar.datetime
self.min60Bar = min60Bar
else:
min60Bar = self.min60Bar
min60Bar.high = max(min60Bar.high, bar.high)
min60Bar.low = min(min60Bar.low, bar.low)
min60Bar.close = bar.close
def generateVtBar(d):
"""生成K线"""
bar = VtBarData()
bar.symbol = d['symbol']
bar.vtSymbol = d['symbol']
bar.date = d['date']
bar.time = ':'.join([d['time'][:2], d['time'][2:]])
bar.open = d['open']
bar.high = d['high']
bar.low = d['low']
bar.close = d['close']
bar.volume = d['volume']
bar.openInterest = d['openInterest']
bar.datetime = datetime.datetime.strptime(' '.join([bar.date, bar.time]), '%Y%m%d %H:%M')
return bar
def downloadEquityDailyBarts(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print u'开始下载%s日行情' %symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
import tushare as ts
if last:
start = last['date'][:4]+'-'+last['date'][4:6]+'-'+last['date'][6:]
data = ts.get_k_data(symbol,start)
if not data.empty:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for index, d in data.iterrows():
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.open = d.get('open')
bar.high = d.get('high')
bar.low = d.get('low')
bar.close = d.get('close')
bar.date = d.get('date').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('volume')
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
def downloadEquityDailyBar(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print u'开始下载%s日行情' %symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
path = 'api/market/getMktEqud.json'
params = {}
params['ticker'] = symbol
if last:
params['beginDate'] = last['date']
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
def loadTdxCsv(fileName, dbName, symbol):
"""将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
date_correct = ""
print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[1])
bar.high = float(d[2])
bar.low = float(d[3])
bar.close = float(d[4])
#通达信的夜盘时间按照新的一天计算，此处将其按照当天日期统计，方便后续查阅
date_temp,time_temp = d[0].strip(' ').replace('\xef\xbb\xbf','').split('-',1)
if time_temp == '15:00':
date_correct = date_temp
if time_temp[:2] == "21" or time_temp[:2] == "22" or time_temp[:2] == "23":
date_temp = date_correct
bar.date = datetime.strptime(date_temp, '%Y/%m/%d').strftime('%Y%m%d')
bar.time = time_temp[:2]+':'+time_temp[3:5]+':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = d[5]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'插入完毕，耗时：%s' % (time()-start)
def generateVtBar(row):
"""生成K线"""
bar = VtBarData()
bar.symbol = row['code']
bar.exchange = ''
bar.vtSymbol = bar.symbol
bar.open = row['open']
bar.high = row['high']
bar.low = row['low']
bar.close = row['close']
bar.volume = row['volume']
bar.datetime = datetime.strptime(row['time_key'], '%Y-%m-%d %H:%M:%S')
bar.date = bar.datetime.strftime("%Y%m%d")
bar.time = bar.datetime.strftime("%H:%M:%S")
return bar
def generateVtBar(row):
"""生成K线"""
bar = VtBarData()
symbol, exchange = row['symbol'].split('.')
bar.symbol = symbol
bar.exchange = exchangeMapReverse[exchange]
bar.vtSymbol = '.'.join([bar.symbol, bar.exchange])
bar.open = row['open']
bar.high = row['high']
bar.low = row['low']
bar.close = row['close']
bar.volume = row['volume']
bar.date = str(row['date'])
bar.time = str(row['time']).rjust(6, '0')
#将bar的时间改成提前一分钟
hour=bar.time[0:2]
minute=bar.time[2:4]
sec=bar.time[4:6]
if minute=="00":
minute="59"
h = int(hour)
if h == 0:
h = 24
hour=str(h-1).rjust(2,'0')
else:
minute=str(int(minute)-1).rjust(2,'0')
bar.time=hour+minute+sec
bar.datetime = datetime.strptime(' '.join([bar.date, bar.time]), '%Y%m%d %H%M%S')
return bar
def loadOKEXCsv(fileName, dbName, symbol):
"""将OKEX导出的csv格式的历史分钟数据插入到Mongo数据库中"""
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(open(fileName,"r"))
for d in reader:
if len(d[1]) > 10:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.datetime = datetime.strptime(d[1], '%Y-%m-%d %H:%M:%S')
bar.date = bar.datetime.date().strftime('%Y%m%d')
bar.time = bar.datetime.time().strftime('%H:%M:%S')
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.volume = float(d[6])
bar.tobtcvolume = float(d[7])
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print('%s \t %s' % (bar.date, bar.time))
print u'插入完毕，耗时：%s' % (time()-start)
def generateVtBar(row):
"""生成K线"""
bar = VtBarData()
bar.symbol = row['code']
bar.exchange = generateExchange(bar.symbol)
bar.vtSymbol = '.'.join([bar.symbol, bar.exchange])
bar.open = row['open']
bar.high = row['high']
bar.low = row['low']
bar.close = row['close']
bar.volume = row['vol']
bar.datetime = row.name
bar.date = bar.datetime.strftime("%Y%m%d")
bar.time = bar.datetime.strftime("%H:%M:%S")
return bar
def loadMcCsv(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
import csv
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.DictReader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d['Open'])
bar.high = float(d['High'])
bar.low = float(d['Low'])
bar.close = float(d['Close'])
bar.date = datetime.strptime(d['Date'], '%Y-%m-%d').strftime('%Y%m%d')
bar.time = d['Time']
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = d['TotalVolume']
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print bar.date, bar.time
print u'插入完毕，耗时：%s' % (time()-start)
def downloadFuturesIntradayBar(self, symbol):
"""下载期货的日内分钟行情"""
print u'开始下载%s日内分钟行情' %symbol
# 日内分钟行情只有具体合约
path = 'api/market/getFutureBarRTIntraDay.json'
params = {}
params['instrumentID'] = symbol
params['unit'] = 1
data = self.datayesClient.downloadData(path, params)
if data:
today = datetime.now().strftime('%Y%m%d')
# 创建datetime索引
self.dbClient[MINUTE_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = today
bar.time = d.get('barTime', '')
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M')
bar.volume = d.get('totalVolume', 0)
bar.openInterest = 0
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[MINUTE_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
def downloadFuturesIntradayBar(self, symbol):
"""下载期货的日内分钟行情"""
print u'开始下载%s日内分钟行情' % symbol
# 日内分钟行情只有具体合约
path = 'api/market/getFutureBarRTIntraDay.json'
params = {}
params['instrumentID'] = symbol
params['unit'] = 1
data = self.datayesClient.downloadData(path, params)
if data:
today = datetime.now().strftime('%Y%m%d')
# 创建datetime索引
self.dbClient[MINUTE_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = today
bar.time = d.get('barTime', '')
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M')
bar.volume = d.get('totalVolume', 0)
bar.openInterest = 0
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[MINUTE_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
def loadTdxLc1(fileName, dbName, symbol):
"""将通达信导出的lc1格式的历史分钟数据插入到Mongo数据库中"""
# from struct import *
start = time()
print('开始读取通达信Lc1文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
#读取二进制文件
ofile = open(fileName, 'rb')
buf = ofile.read()
ofile.close()
num = len(buf)
no = num / 32
b = 0
e = 32
dl = []
for i in range(no):
a = unpack('hhfffffii', buf[b:e])
b = b + 32
e = e + 32
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = a[2]
bar.high = a[3]
bar.low = a[4]
bar.close = a[5]
bar.date = str(int(a[0] / 2048) + 2004) + str(int(
a[0] % 2048 / 100)).zfill(2) + str(a[0] % 2048 % 100).zfill(2)
bar.time = str(int(a[1] / 60)).zfill(2) + ':' + str(
a[1] % 60).zfill(2) + ':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = a[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print('插入完毕，耗时：%s' % (time() - start))
def loadTdxCsv(fileName, dbName, symbol):
"""将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.date = datetime.strptime(d[0], '%Y/%m/%d').strftime('%Y%m%d')
bar.time = d[1][:2] + ':' + d[1][2:4] + ':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d[6]
bar.openInterest = d[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print bar.date, bar.time
print u'插入完毕，耗时：%s' % (time() - start)
def loadMcCsv(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
import csv
start = time()
print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.DictReader(open(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d['Open'])
bar.high = float(d['High'])
bar.low = float(d['Low'])
bar.close = float(d['Close'])
bar.date = datetime.strptime(d['Date'], '%Y-%m-%d').strftime('%Y%m%d')
bar.time = d['Time']
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d['TotalVolume']
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print(bar.date, bar.time)
print('插入完毕，耗时：%s' % (time() - start))
def generateVtBar(symbol, time, d):
"""生成K线"""
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d['open'])
bar.high = float(d['high'])
bar.low = float(d['low'])
bar.close = float(d['close'])
bar.date = datetime.strptime(time[0:10], '%Y-%m-%d').strftime('%Y%m%d')
bar.time = time[11:]
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d['volume']
return bar
def generateVtBar(row, symbol):
"""生成K线"""
bar = VtBarData()
bar.symbol = symbol
bar.vtSymbol = symbol
bar.open = row['open']
bar.high = row['high']
bar.low = row['low']
bar.close = row['close']
bar.volume = row['volume']
bar.datetime = row.name
bar.date = bar.datetime.strftime("%Y%m%d")
bar.time = bar.datetime.strftime("%H:%M:%S")
return bar
print u'数据下载完成'
# 创建MongoDB连接
client = pymongo.MongoClient('localhost', 27017)
collection = client[DAILY_DB_NAME][vtSymbol]
collection.create_index('datetime')
print u'MongoDB连接成功'
# 将数据插入历史数据库
for row in data.iterrows():
date = row[0]
data = row[1]
bar = VtBarData()
bar.vtSymbol = vtSymbol
bar.symbol = symbol
bar.exchange = exchange
bar.date = date
bar.datetime = datetime.strptime(date, '%Y-%m-%d')
bar.open = data['open']
bar.high = data['high']
bar.low = data['low']
bar.close = data['close']
bar.volume = data['volume']
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'数据插入完成'
def onTick(self, tick):
"""收到行情TICK推送（必须由用户继承实现）"""
# 计算K线
tickMinute = tick.datetime.minute
if tickMinute != self.barMinute:
if self.bar:
self.onBar(self.bar)
bar = VtBarData()
bar.vtSymbol = tick.vtSymbol
bar.symbol = tick.symbol
bar.exchange = tick.exchange
bar.open = tick.lastPrice
bar.high = tick.lastPrice
bar.low = tick.lastPrice
bar.close = tick.lastPrice
bar.date = tick.date
bar.time = tick.time
bar.datetime = tick.datetime # K线的时间设为第一个Tick的时间
self.bar = bar # 这种写法为了减少一层访问，加快速度
self.barMinute = tickMinute # 更新当前的分钟
else: # 否则继续累加新的K线
bar = self.bar # 写法同样为了加快速度
bar.high = max(bar.high, tick.lastPrice)
bar.low = min(bar.low, tick.lastPrice)
bar.close = tick.lastPrice
def loadTbPlusCsv(fileName, dbName, symbol):
"""将TB极速版导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.date = str(d[0])
tempstr = str(round(float(d[1]) * 10000)).split(".")[0].zfill(4)
bar.time = tempstr[:2] + ":" + tempstr[2:4] + ":00"
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d[6]
bar.openInterest = d[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print(bar.date, bar.time)
print('插入完毕，耗时：%s' % (time() - start))
def generateVtBar(symbol, d):
"""生成K线"""
bar = VtBarData()
bar.symbol = symbol
bar.vtSymbol = symbol
bar.open = d['open']
bar.high = d['high']
bar.low = d['low']
bar.close = d['close']
bar.volume = d['volume']
bar.openInterest = d['open_oi']
bar.datetime = datetime.fromtimestamp(d['datetime']/1000000000)
bar.date = bar.datetime.strftime("%Y%m%d")
bar.time = bar.datetime.strftime("%H:%M:%S")
return bar
def downloadEquityDailyBarts(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print('开始下载%s日行情' % symbol)
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
import tushare as ts
if last:
start = last['date'][:4] + '-' + last['date'][4:6] + '-' + last[
'date'][6:]
data = ts.get_k_data(symbol, start)
if not data.empty:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for index, d in data.iterrows():
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.open = d.get('open')
bar.high = d.get('high')
bar.low = d.get('low')
bar.close = d.get('close')
bar.date = d.get('date').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('volume')
except KeyError:
print(d)
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print('%s下载完成' % symbol)
else:
print('找不到合约%s' % symbol)
def loadRqBar(self, symbol, days):
"""从RQData加载K线数据"""
endDate = datetime.now()
startDate = endDate - timedelta(days)
df = self.rq.get_price(symbol.upper(),
frequency='1m',
fields=['open', 'high', 'low', 'close', 'volume'],
start_date=startDate,
end_date=endDate)
l = []
for ix, row in df.iterrows():
bar = VtBarData()
bar.symbol = symbol
bar.vtSymbol = symbol
bar.open = row['open']
bar.high = row['high']
bar.low = row['low']
bar.close = row['close']
bar.volume = row['volume']
bar.datetime = row.name
bar.date = bar.datetime.strftime("%Y%m%d")
bar.time = bar.datetime.strftime("%H:%M:%S")
l.append(bar)
return l
def loadOKCsv(fileName, dbName, symbol):
"""将OKEX导出的csv格式的历史分钟数据插入到Mongo数据库中"""
start = time()
print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(open(fileName, "r"))
for d in reader:
print(d)
if len(d[0]) > 10:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol, bar.exchange = symbol.split(':')
bar.datetime = datetime.strptime(d[0], '%Y-%m-%d %H:%M:%S')
bar.date = bar.datetime.date().strftime('%Y%m%d')
bar.time = bar.datetime.time().strftime('%H:%M:%S')
print(d[1])
if d[1]:
bar.high = float(d[1])
bar.low = float(d[2])
bar.open = float(d[3])
bar.close = float(d[4])
bar.amount = float(d[5])
bar.volume = float(d[6])
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print('%s \t %s' % (bar.date, bar.time))
print('插入完毕，耗时：%s' % (time() - start))
def generateVtBar(symbol, d):
"""生成K线"""
bar = VtBarData()
bar.symbol = symbol
bar.vtSymbol = symbol
bar.datetime = datetime.datetime.strptime(d['time_open'], '%Y-%m-%dT%H:%M:%S.%f0Z')
bar.date = bar.datetime.strftime('%Y%m%d')
bar.time = bar.datetime.strftime('%H:%M:%S')
bar.open = d['price_open']
bar.high = d['price_high']
bar.low = d['price_low']
bar.close = d['price_close']
bar.volume = d['volume_traded']
return bar
def generateBar(self, row, symbol):
"""生成K线对象"""
bar = VtBarData()
bar.symbol = symbol
bar.vtSymbol = symbol
bar.open = row['open']
bar.high = row['high']
bar.low = row['low']
bar.close = row['close']
bar.volume = row['volume']
bar.datetime = row.name
bar.date = bar.datetime.strftime("%Y%m%d")
bar.time = bar.datetime.strftime("%H:%M:%S")
return bar
def onBar(self, bar):
"""收到Bar推送（必须由用户继承实现）"""
# 如果当前是一个5分钟走完（分钟线的时间戳是当前分钟的开始时间戳，因此要+1）
if (bar.datetime.minute + 1) % 5 == 0:
# 如果已经有聚合5分钟K线
if self.fiveBar:
# 将最新分钟的数据更新到目前5分钟线中
fiveBar = self.fiveBar
fiveBar.high = max(fiveBar.high, bar.high)
fiveBar.low = min(fiveBar.low, bar.low)
fiveBar.close = bar.close
# 推送5分钟线数据
self.onFiveBar(fiveBar)
# 清空5分钟线数据缓存
self.fiveBar = None
else:
# 如果没有缓存则新建
if not self.fiveBar:
fiveBar = VtBarData()
fiveBar.vtSymbol = bar.vtSymbol
fiveBar.symbol = bar.symbol
fiveBar.exchange = bar.exchange
fiveBar.open = bar.open
fiveBar.high = bar.high
fiveBar.low = bar.low
fiveBar.close = bar.close
fiveBar.date = bar.date
fiveBar.time = bar.time
fiveBar.datetime = bar.datetime
self.fiveBar = fiveBar
else:
fiveBar = self.fiveBar
fiveBar.high = max(fiveBar.high, bar.high)
fiveBar.low = min(fiveBar.low, bar.low)
fiveBar.close = bar.close