def checkInformationData(self):
"""Update information symbols' data"""
# If infobar is empty, which means it is the first time calling this method
if self.infobar == {}:
for info_symbol in self.InfoCursor:
try:
self.infobar[info_symbol] = next(
self.InfoCursor[info_symbol])
except StopIteration:
print(
"Data of information symbols is empty! Input must be a list, not str."
)
raise
temp = {}
for info_symbol in self.infobar:
data = self.infobar[info_symbol]
# Update data only when Time Stamp is matched
if (data is not None) and (data['datetime'] <= self.dt):
try:
temp[info_symbol] = VtBarData()
temp[info_symbol].__dict__ = data
self.infobar[info_symbol] = next(
self.InfoCursor[info_symbol])
except StopIteration:
self.infobar[info_symbol] = None
self.output("No more data in information database.")
else:
temp[info_symbol] = None
return temp
def loadBar(self, dbName, collectionName, days):
"""从数据库中读取Bar数据，startDate是datetime对象"""
startDate = self.today - timedelta(days)
d = {'datetime': {'$gte': startDate}}
barData = self.mainEngine.dbQuery(dbName, collectionName, d)
l = []
for d in barData:
bar = VtBarData()
bar.__dict__ = d
l.append(bar)
return l
def tickDataProcess(self, tick):
"""收到行情TICK推送, 历史tick"""
print('tickDataProcess, time:', tick.time)
struct_time = time.strptime(tick.time, "%H:%M:%S")
tickMinute = struct_time.tm_min
bar = self.bar
if tickMinute != self.barMinute:
if bar:
# add to list
self.dateList.append(bar.date)
self.openList.append(bar.open)
self.closeList.append(bar.close)
self.highList.append(bar.high)
self.lowList.append(bar.low)
self.volumeList.append(bar.volume)
self.onMin1Bar(self.bar)
else:
# the first
# new bar data
self.bar = VtBarData()
bar = self.bar
#bar.vtSymbol = tick.vtSymbol
#bar.symbol = tick.symbol
#bar.exchange = tick.exchange
bar.open = tick.lastPrice
bar.high = tick.lastPrice
bar.low = tick.lastPrice
bar.close = tick.lastPrice
bar.volume = 0
bar.date = tick.date
bar.time = tick.time
bar.datetime = tick.datetime # K线的时间设为第一个Tick的时间
self.barMinute = tickMinute # 更新当前的分钟
else: # 否则继续累加新的K线
bar.high = max(bar.high, tick.lastPrice)
bar.low = min(bar.low, tick.lastPrice)
bar.close = tick.lastPrice
bar.volume = bar.volume + int(tick.volume)
def __firstTick(self, tick):
""" K线的第一个Tick数据"""
self.bar = VtBarData() # 创建新的K线
self.bar.vtSymbol = tick.vtSymbol
self.bar.symbol = tick.symbol
self.bar.exchange = tick.exchange
self.bar.open = tick.lastPrice # O L H C
self.bar.high = tick.lastPrice
self.bar.low = tick.lastPrice
self.bar.close = tick.lastPrice
# K线的日期时间
self.bar.date = tick.date # K线的日期时间（去除秒）设为第一个Tick的时间
self.bar.time = tick.time # K线的日期时间（去除秒）设为第一个Tick的时间
self.bar.datetime = tick.datetime
self.bar.volume = tick.volume
self.bar.openInterest = tick.openInterest
self.barFirstTick = True # 标识该Tick属于该Bar的第一个tick数据
self.lineBar.append(self.bar) # 推入到lineBar队列
def downloadFuturesDailyBar(self, symbol):
"""
下载期货合约的日行情，symbol是合约代码，
若最后四位为0000（如IF0000），代表下载连续合约。
"""
print('开始下载%s日行情' % symbol)
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 主力合约
if '0000' in symbol:
path = 'api/market/getMktMFutd.json'
params = {}
params['contractObject'] = symbol.replace('0000', '')
params['mainCon'] = 1
if last:
params['startDate'] = last['date']
# 交易合约
else:
path = 'api/market/getMktFutd.json'
params = {}
params['ticker'] = symbol
if last:
params['startDate'] = last['date']
# 开始下载数据
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
bar.openInterest = d.get('openInt', 0)
except KeyError:
print(d)
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print('%s下载完成' % symbol)
else:
print('找不到合约%s' % symbol)
def loadTdxCsv(fileName, dbName, symbol):
"""将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.date = datetime.strptime(d[0], '%Y/%m/%d').strftime('%Y%m%d')
bar.time = d[1][:2] + ':' + d[1][2:4] + ':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d[6]
bar.openInterest = d[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print(bar.date, bar.time)
print('插入完毕，耗时：%s' % (time() - start))
def loadMcCsv(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
import csv
start = time()
print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.DictReader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d['Open'])
bar.high = float(d['High'])
bar.low = float(d['Low'])
bar.close = float(d['Close'])
bar.date = datetime.strptime(d['Date'], '%Y-%m-%d').strftime('%Y%m%d')
bar.time = d['Time']
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d['TotalVolume']
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print(bar.date, bar.time)
print('插入完毕，耗时：%s' % (time() - start))
def downloadEquityDailyBarts(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print('开始下载%s日行情' % symbol)
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
import tushare as ts
if last:
start = last['date'][:4] + '-' + last['date'][4:6] + '-' + last[
'date'][6:]
data = ts.get_k_data(symbol, start)
if not data.empty:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for index, d in data.iterrows():
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.open = d.get('open')
bar.high = d.get('high')
bar.low = d.get('low')
bar.close = d.get('close')
bar.date = d.get('date').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('volume')
except KeyError:
print(d)
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print('%s下载完成' % symbol)
else:
print('找不到合约%s' % symbol)
def downloadEquityDailyBar(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print('开始下载%s日行情' % symbol)
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
path = 'api/market/getMktEqud.json'
params = {}
params['ticker'] = symbol
if last:
params['beginDate'] = last['date']
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
except KeyError:
print(d)
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print('%s下载完成' % symbol)
else:
print('找不到合约%s' % symbol)
def downloadFuturesIntradayBar(self, symbol):
"""下载期货的日内分钟行情"""
print('开始下载%s日内分钟行情' % symbol)
# 日内分钟行情只有具体合约
path = 'api/market/getFutureBarRTIntraDay.json'
params = {}
params['instrumentID'] = symbol
params['unit'] = 1
data = self.datayesClient.downloadData(path, params)
if data:
today = datetime.now().strftime('%Y%m%d')
# 创建datetime索引
self.dbClient[MINUTE_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = today
bar.time = d.get('barTime', '')
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M')
bar.volume = d.get('totalVolume', 0)
bar.openInterest = 0
except KeyError:
print(d)
flt = {'datetime': bar.datetime}
self.dbClient[MINUTE_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print('%s下载完成' % symbol)
else:
print('找不到合约%s' % symbol)
def onTick(self, tick):
"""收到行情TICK推送（必须由用户继承实现）"""
# 计算K线
tickMinute = tick.datetime.minute
if tickMinute != self.barMinute:
if self.bar:
self.onBar(self.bar)
bar = VtBarData()
bar.vtSymbol = tick.vtSymbol
bar.symbol = tick.symbol
bar.exchange = tick.exchange
bar.open = tick.lastPrice
bar.high = tick.lastPrice
bar.low = tick.lastPrice
bar.close = tick.lastPrice
bar.date = tick.date
bar.time = tick.time
bar.datetime = tick.datetime # K线的时间设为第一个Tick的时间
self.bar = bar # 这种写法为了减少一层访问，加快速度
self.barMinute = tickMinute # 更新当前的分钟
else: # 否则继续累加新的K线
bar = self.bar # 写法同样为了加快速度
bar.high = max(bar.high, tick.lastPrice)
bar.low = min(bar.low, tick.lastPrice)
bar.close = tick.lastPrice
def loadSetting(self):
"""载入设置"""
with open(self.settingFileName) as f:
drSetting = json.load(f)
# 如果working设为False则不启动行情记录功能
working = drSetting['working']
if not working:
return
if 'tick' in drSetting:
l = drSetting['tick']
for setting in l:
symbol = setting[0]
vtSymbol = symbol
req = VtSubscribeReq()
req.symbol = setting[0]
# 针对LTS和IB接口，订阅行情需要交易所代码
if len(setting)>=3:
req.exchange = setting[2]
vtSymbol = '.'.join([symbol, req.exchange])
# 针对IB接口，订阅行情需要货币和产品类型
if len(setting)>=5:
req.currency = setting[3]
req.productClass = setting[4]
self.mainEngine.subscribe(req, setting[1])
tick = VtTickData() # 该tick实例可以用于缓存部分数据（目前未使用）
self.tickDict[vtSymbol] = tick
if 'bar' in drSetting:
l = drSetting['bar']
for setting in l:
symbol = setting[0]
vtSymbol = symbol
req = VtSubscribeReq()
req.symbol = symbol
if len(setting)>=3:
req.exchange = setting[2]
vtSymbol = '.'.join([symbol, req.exchange])
if len(setting)>=5:
req.currency = setting[3]
req.productClass = setting[4]
self.mainEngine.subscribe(req, setting[1])
bar = VtBarData()
self.barDict[vtSymbol] = bar
if 'active' in drSetting:
d = drSetting['active']
# 注意这里的vtSymbol对于IB和LTS接口，应该后缀.交易所
for activeSymbol, vtSymbol in list(d.items()):
self.activeSymbolDict[vtSymbol] = activeSymbol
# 启动数据插入线程
self.start()
# 注册事件监听
self.registerEvent()
def onBar(self, bar):
"""收到Bar推送（必须由用户继承实现）"""
# 如果当前是一个5分钟走完
if bar.datetime.minute % 5 == 0:
# 如果已经有聚合5分钟K线
if self.fiveBar:
# 将最新分钟的数据更新到目前5分钟线中
fiveBar = self.fiveBar
fiveBar.high = max(fiveBar.high, bar.high)
fiveBar.low = min(fiveBar.low, bar.low)
fiveBar.close = bar.close
# 推送5分钟线数据
self.onFiveBar(fiveBar)
# 清空5分钟线数据缓存
self.fiveBar = None
else:
# 如果没有缓存则新建
if not self.fiveBar:
fiveBar = VtBarData()
fiveBar.vtSymbol = bar.vtSymbol
fiveBar.symbol = bar.symbol
fiveBar.exchange = bar.exchange
fiveBar.open = bar.open
fiveBar.high = bar.high
fiveBar.low = bar.low
fiveBar.close = bar.close
fiveBar.date = bar.date
fiveBar.time = bar.time
fiveBar.datetime = bar.datetime
self.fiveBar = fiveBar
else:
fiveBar = self.fiveBar
fiveBar.high = max(fiveBar.high, bar.high)
fiveBar.low = min(fiveBar.low, bar.low)
fiveBar.close = bar.close