def processLine(self, line):
historyData = line.split(',')
#historyDataLen = len(historyData)
symbol = historyData[2]
#print 'processLine, symbol:' + symbol
#从list转化为tick对象
historytick = VtTickData()
historytick._id = historyData[0]
historytick.gatewayName = 'CTP'
historytick.symbol = symbol
historytick.TradingDay = historyData[1]
historytick.exchange = historyData[3]
historytick.vtSymbol = historytick.symbol # '.'.join([tick.symbol, tick.exchange])
historytick.lastPrice = self.convertFloatZero(historyData[5])
#lastVolume
historytick.volume = historyData[12]
historytick.openInterest = historyData[14]
UpdateMillisec = int(historyData[20])
historytick.time = '.'.join([historyData[19], str(UpdateMillisec/ 100)])
historytick.date = historyData[42]
historytick.datetime = datetime.strptime(' '.join([historytick.date, historytick.time]), '%Y%m%d %H:%M:%S.%f')
historytick.openPrice = self.convertFloatZero(historyData[9])
historytick.highPrice = self.convertFloatZero(historyData[10])
historytick.lowPrice = self.convertFloatZero(historyData[11])
historytick.preClosePrice = self.convertFloatZero(historyData[12])
historytick.ClosePrice = self.convertFloatZero(historyData[15])
historytick.SettlementPrice = self.convertFloatZero(historyData[16])
historytick.upperLimit = self.convertFloatZero(historyData[17])
historytick.lowerLimit = self.convertFloatZero(historyData[18])
# CTP只有一档行情
historytick.bidPrice1 = self.convertFloatZero(historyData[21])
historytick.bidVolume1 = historyData[22]
historytick.askPrice1 = self.convertFloatZero(historyData[23])
historytick.askVolume1 = historyData[24]
historytick.AveragePrice = self.convertFloatZero(historyData[41])
return historytick
def loadqlCsv(fileName, dbName, symbol):
"""将TB极速版导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
count = 0
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
tick = VtTickData()
tick.vtSymbol = symbol
tick.symbol = symbol
tick.lastPrice = float(d[5])
tick.volume = int(d[6])
tick.openPrice = float(d[9])
tick.highPrice = float(d[7])
tick.lowPrice = float(d[8])
tick.preClosePrice = float(d[20])
tick.askPrice1 = float(d[12])
tick.bidPrice1 = float(d[14])
tick.askVolume1 = float(d[13])
tick.bidVolume1 = float(d[15])
count += 1
tick.date = datetime.strptime(d[2], '%Y%m%d').strftime('%Y%m%d')
if d[4] == '500':
tick.time = d[3] + "." + "500000"
else:
tick.time = d[3] + "." + "000000"
tick.datetime = datetime.strptime(tick.date + ' ' + tick.time,
'%Y%m%d %H:%M:%S.%f')
flt = {'datetime': tick.datetime}
collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
print u'插入完毕，耗时：%s插入数量' % (time() - start), count
def generateVtTick(row, symbol):
"""生成K线"""
tick = VtTickData()
tick.symbol = symbol
tick.vtSymbol = symbol
tick.lastPrice = row['last']
tick.volume = row['volume']
tick.openInterest = row['open_interest']
tick.datetime = row.name
tick.openPrice = row['open']
tick.highPrice = row['high']
tick.lowPrice = row['low']
tick.preClosePrice = row['prev_close']
tick.upperLimit = row['limit_up']
tick.lowerLimit = row['limit_down']
tick.bidPrice1 = row['b1']
tick.bidPrice2 = row['b2']
tick.bidPrice3 = row['b3']
tick.bidPrice4 = row['b4']
tick.bidPrice5 = row['b5']
tick.bidVolume1 = row['b1_v']
tick.bidVolume2 = row['b2_v']
tick.bidVolume3 = row['b3_v']
tick.bidVolume4 = row['b4_v']
tick.bidVolume5 = row['b5_v']
tick.askPrice1 = row['a1']
tick.askPrice2 = row['a2']
tick.askPrice3 = row['a3']
tick.askPrice4 = row['a4']
tick.askPrice5 = row['a5']
tick.askVolume1 = row['a1_v']
tick.askVolume2 = row['a2_v']
tick.askVolume3 = row['a3_v']
tick.askVolume4 = row['a4_v']
tick.askVolume5 = row['a5_v']
return tick
def generateVtTick(row, symbol):
"""生成K线"""
tick = VtTickData()
tick.symbol = symbol
tick.vtSymbol = symbol
tick.lastPrice = row['last']
tick.volume = row['volume']
tick.openInterest = row['open_interest']
tick.datetime = row.name
tick.openPrice = row['open']
tick.highPrice = row['high']
tick.lowPrice = row['low']
tick.preClosePrice = row['prev_close']
tick.upperLimit = row['limit_up']
tick.lowerLimit = row['limit_down']
tick.bidPrice1 = row['b1']
tick.bidPrice2 = row['b2']
tick.bidPrice3 = row['b3']
tick.bidPrice4 = row['b4']
tick.bidPrice5 = row['b5']
tick.bidVolume1 = row['b1_v']
tick.bidVolume2 = row['b2_v']
tick.bidVolume3 = row['b3_v']
tick.bidVolume4 = row['b4_v']
tick.bidVolume5 = row['b5_v']
tick.askPrice1 = row['a1']
tick.askPrice2 = row['a2']
tick.askPrice3 = row['a3']
tick.askPrice4 = row['a4']
tick.askPrice5 = row['a5']
tick.askVolume1 = row['a1_v']
tick.askVolume2 = row['a2_v']
tick.askVolume3 = row['a3_v']
tick.askVolume4 = row['a4_v']
tick.askVolume5 = row['a5_v']
return tick
def startWork(self):
for root, subdirs, files in os.walk(self.walkingDir):
for theFile in files:
currentDate = None
currentSymbol = ''
count = 0
# 确定合约【由文件夹名称决定，因为主力合约只能在文件夹名称中确定】
if '\\' in root:
currentSymbol = root.split('\\')[-1]
currentSymbol = currentSymbol
# 排除不合法文件
if theFile.startswith('.'):
continue
# 确定日期
if '.' in theFile:
pos = theFile.index('.')
dateStr = theFile[:pos]
currentDate = datetime.datetime.strptime(dateStr, '%Y%m%d')
# 合约不存在
if not currentSymbol:
print('*' * 6)
print(u'没有找到合约')
print('*' * 6)
exit(0)
# 检查是否有数据库记录
if currentDate and (not self.needUpdateChecking(
currentSymbol, currentDate)):
continue
if '.csv' in theFile:
# 读取文件
filePath = root + '\\' + theFile
with open(filePath, 'r') as f:
reader = csv.DictReader(f)
# 开始导入数据
startTime = time()
for row in reader:
count += 1
if count == 1:
print('=' * 6, currentSymbol, ' ',
currentDate.date(), '=' * 6)
# 过滤无效数据
timeStr = row['time']
theLen = len(timeStr)
if theLen < 9:
sub = 9 - theLen
timeStr = '0' * sub + timeStr
'''
hour = int(timeStr[0:2])
minute = int(timeStr[2:4])
second = int(timeStr[4:6])
microSecond = int(timeStr[6:9])
if hour < 0 or hour > 59 or minute < 0 or minute > 59 or second < 0 or second > 59 or microSecond < 0 or microSecond > 999:
print u'删除时间格式错误数据：%s' % timeStr
continue
'''
try:
tickTime = datetime.datetime.strptime(
timeStr, '%H%M%S%f')
except Exception:
print(u'时间转换异常：%s' % timeStr)
raise
exit(0)
t = tickTime.time()
fakeData = True
if self.dataType == 1:
# 商品期货
if self.night == 0:
# 无夜盘交易
if ((MORNING_START_CF <= t <
MORNING_REST_CF)
or (MORNING_RESTART_CF <= t <
MORNING_END_CF)
or (AFTERNOON_START_CF <= t <
AFTERNOON_END_CF)):
fakeData = False
elif self.night == 1:
# 夜间收盘
if ((MORNING_START_CF <= t <
MORNING_REST_CF)
or (MORNING_RESTART_CF <= t <
MORNING_END_CF)
or (AFTERNOON_START_CF <= t <
AFTERNOON_END_CF)
or (NIGHT_START_CF <= t <
NIGHT_END_CF_N)):
fakeData = False
elif self.night == 2:
# 凌晨收盘
if ((MORNING_START_CF <= t <
MORNING_REST_CF)
or (MORNING_RESTART_CF <= t <
MORNING_END_CF)
or (AFTERNOON_START_CF <= t <
AFTERNOON_END_CF)
or (NIGHT_START_CF <= t)
or (NIGHT_END_CF_M > t)):
fakeData = False
elif self.dataType == 2:
# 股指期货
if ((MORNING_START_SF <= t < MORNING_END_SF)
or (AFTERNOON_START_SF <= t <
AFTERNOON_END_SF)):
fakeData = False
theDate = row['calendarday']
theTime = tickTime.strftime('%H:%M:%S.%f')
theDatetime = datetime.datetime.strptime(
theDate + ' ' + theTime, '%Y%m%d %H:%M:%S.%f')
# 数据库collection
collection = self.db[currentSymbol]
collection.create_index('datetime')
if fakeData:
print(t)
print(u'【剔除无效数据】')
# 删除已保存的数据库数据
flt = {'datetime': theDatetime}
cursor = collection.find(flt)
if cursor.count():
print(u'【剔除无效‘数据库’数据】')
collection.delete_many(flt)
continue
# 创建VtTickData对象
tick = VtTickData()
tick.vtSymbol = row['instrument']
tick.lastPrice = float(row['lastp'])
tick.volume = int(row['volume'])
tick.openInterest = int(row['openinterest'])
tick.askPrice1 = float(row['ask1'])
tick.askVolume1 = int(row['asksz1'])
tick.bidPrice1 = float(row['bid1'])
tick.bidVolume1 = int(row['bidsz1'])
tick.date = theDate
tick.time = theTime
tick.datetime = theDatetime
tick.tradingDay = row['tradingday']
# 保存tick到数据库
collection.update_many({'datetime': tick.datetime},
{'$set': tick.__dict__},
upsert=True)
# 打印进程
if count:
sub = time() - startTime
print(u'用时：', sub, 's')
print(u'数据量：', count, '\n')
# 保存记录
if currentDate and count:
self.updateHistory(currentSymbol, currentDate)
print(u'所有数据导入完成')
def add_gap_ticks(self):
"""
补充缺失的分时数据
:return:
"""
ds = DataSource()
for vtSymbol in self.renkoDict.keys():
renkobar_list = self.renkoDict.get(vtSymbol, [])
cache_bars = None
cache_start_date = None
cache_end_date = None
for renko_bar in renkobar_list:
# 通过mongo获取最新一个Renko bar的数据日期close时间
last_renko_dt = self.get_last_datetime(renko_bar.name)
# 根据日期+vtSymbol，像datasource获取分钟数据，以close价格，转化为tick，推送到renko_bar中
if last_renko_dt is not None:
start_date = last_renko_dt.strftime('%Y-%m-%d')
else:
start_date = (datetime.now() -
timedelta(days=90)).strftime('%Y-%m-%d')
end_date = (datetime.now() +
timedelta(days=5)).strftime('%Y-%m-%d')
self.writeDrLog(u'从datasource获取{}数据,开始日期:{}'.format(
vtSymbol, start_date))
if cache_bars is None or cache_start_date != start_date or cache_end_date != end_date:
fields = [
'open', 'close', 'high', 'low', 'volume',
'open_interest', 'limit_up', 'limit_down',
'trading_date'
]
cache_bars = ds.get_price(order_book_id=vtSymbol,
start_date=start_date,
end_date=end_date,
frequency='1m',
fields=fields)
cache_start_date = start_date
cache_end_date = end_date
if cache_bars is not None:
total = len(cache_bars)
self.writeDrLog(u'一共获取{}条{} 1分钟数据'.format(
total, vtSymbol))
if cache_bars is not None:
self.writeDrLog(u'推送分时数据tick:{}到:{}'.format(
vtSymbol, renko_bar.name))
for idx in cache_bars.index:
row = cache_bars.loc[idx]
tick = VtTickData()
tick.vtSymbol = vtSymbol
tick.symbol = vtSymbol
last_bar_dt = datetime.strptime(
str(idx), '%Y-%m-%d %H:%M:00')
tick.datetime = last_bar_dt - timedelta(minutes=1)
tick.date = tick.datetime.strftime('%Y-%m-%d')
tick.time = tick.datetime.strftime('%H:%M:00')
if tick.datetime.hour >= 21:
if tick.datetime.isoweekday() == 5:
# 星期五=》星期一
tick.tradingDay = (
tick.datetime +
timedelta(days=3)).strftime('%Y-%m-%d')
else:
# 第二天
tick.tradingDay = (
tick.datetime +
timedelta(days=1)).strftime('%Y-%m-%d')
elif tick.datetime.hour < 8 and tick.datetime.isoweekday(
) == 6:
# 星期六=>星期一
tick.tradingDay = (
tick.datetime +
timedelta(days=2)).strftime('%Y-%m-%d')
else:
tick.tradingDay = tick.date
tick.upperLimit = float(row['limit_up'])
tick.lowerLimit = float(row['limit_down'])
tick.lastPrice = float(row['close'])
tick.askPrice1 = float(row['close'])
tick.bidPrice1 = float(row['close'])
tick.volume = int(row['volume'])
tick.askVolume1 = tick.volume
tick.bidVolume1 = tick.volume
if last_renko_dt is not None and tick.datetime <= last_renko_dt:
continue
renko_bar.onTick(tick)