def loadTickData(self, tick): tickData = VtTickData() tickData.lastPrice = float(tick['lastPrice']) tickData.date = tick['date'] tickData.time = tick['time'] tickData.datetime = datetime.strptime( ' '.join([tick['date'], tick['time']]), '%Y%m%d %H:%M:%S.%f') tickData.volume = int(tick['volume']) tickData.vtSymbol = tick['vtSymbol'] tickData.symbol = tick['symbol'] tickData.exchange = tick['exchange'] tickData.openInterest = int(tick['openInterest']) return tickData
def mongodbRowToTick(row_tick): tickData = VtTickData() tickData.lastPrice = float(row_tick['lastPrice']) tickData.date = row_tick['date'] tickData.time = row_tick['time'] tickData.datetime = datetime.strptime( ' '.join([row_tick['date'], row_tick['time']]), '%Y%m%d %H:%M:%S.%f') tickData.volume = int(row_tick['volume']) tickData.vtSymbol = row_tick['vtSymbol'] tickData.symbol = row_tick['symbol'] tickData.exchange = row_tick['exchange'] tickData.TradingDay = row_tick['TradingDay'] tickData.openInterest = int(row_tick['openInterest']) return tickData
def processLine(self, line): historyData = line.split(',') #historyDataLen = len(historyData) symbol = historyData[2] #print 'processLine, symbol:' + symbol #从list转化为tick对象 historytick = VtTickData() historytick._id = historyData[0] historytick.gatewayName = 'CTP' historytick.symbol = symbol historytick.TradingDay = historyData[1] historytick.exchange = historyData[3] historytick.vtSymbol = historytick.symbol # '.'.join([tick.symbol, tick.exchange]) historytick.lastPrice = self.convertFloatZero(historyData[5]) #lastVolume historytick.volume = historyData[12] historytick.openInterest = historyData[14] UpdateMillisec = int(historyData[20]) historytick.time = '.'.join([historyData[19], str(UpdateMillisec/ 100)]) historytick.date = historyData[42] historytick.datetime = datetime.strptime(' '.join([historytick.date, historytick.time]), '%Y%m%d %H:%M:%S.%f') historytick.openPrice = self.convertFloatZero(historyData[9]) historytick.highPrice = self.convertFloatZero(historyData[10]) historytick.lowPrice = self.convertFloatZero(historyData[11]) historytick.preClosePrice = self.convertFloatZero(historyData[12]) historytick.ClosePrice = self.convertFloatZero(historyData[15]) historytick.SettlementPrice = self.convertFloatZero(historyData[16]) historytick.upperLimit = self.convertFloatZero(historyData[17]) historytick.lowerLimit = self.convertFloatZero(historyData[18]) # CTP只有一档行情 historytick.bidPrice1 = self.convertFloatZero(historyData[21]) historytick.bidVolume1 = historyData[22] historytick.askPrice1 = self.convertFloatZero(historyData[23]) historytick.askVolume1 = historyData[24] historytick.AveragePrice = self.convertFloatZero(historyData[41]) return historytick
def loadqlCsv(fileName, dbName, symbol): """将TB极速版导出的csv格式的历史分钟数据插入到Mongo数据库中""" import csv count = 0 start = time() print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol) # 锁定集合,并创建索引 client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort']) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) # 读取数据和插入到数据库 reader = csv.reader(file(fileName, 'r')) for d in reader: tick = VtTickData() tick.vtSymbol = symbol tick.symbol = symbol tick.lastPrice = float(d[5]) tick.volume = int(d[6]) tick.openPrice = float(d[9]) tick.highPrice = float(d[7]) tick.lowPrice = float(d[8]) tick.preClosePrice = float(d[20]) tick.askPrice1 = float(d[12]) tick.bidPrice1 = float(d[14]) tick.askVolume1 = float(d[13]) tick.bidVolume1 = float(d[15]) count += 1 tick.date = datetime.strptime(d[2], '%Y%m%d').strftime('%Y%m%d') if d[4] == '500': tick.time = d[3] + "." + "500000" else: tick.time = d[3] + "." + "000000" tick.datetime = datetime.strptime(tick.date + ' ' + tick.time, '%Y%m%d %H:%M:%S.%f') flt = {'datetime': tick.datetime} collection.update_one(flt, {'$set': tick.__dict__}, upsert=True) print u'插入完毕,耗时:%s插入数量' % (time() - start), count
def loadTickData(self, tick): tickData = VtTickData() #tickData.lastPrice = float(tick['lastPrice']) #要处理小数位数,只保留一位小数 #线上录得数据写文件,比如原始数据是个整数3967,写到文件里的会是3967.000000000001 tickData.lastPrice = round(float(tick['lastPrice']), 1) tickData.date = tick['date'] tickData.time = tick['time'] tickData.datetime = datetime.strptime(' '.join([tick['date'], tick['time']]), '%Y%m%d %H:%M:%S.%f') tickData.volume = int(tick['volume']) tickData.vtSymbol = tick['vtSymbol'] tickData.symbol = tick['symbol'] tickData.exchange = tick['exchange'] tickData.TradingDay = tick['TradingDay'] #tickData.openInterest = int(tick['openInterest']) #防止持仓量可能带小数 tickData.openInterest = int(float(tick['openInterest'])) return tickData
def startWork(self): for root, subdirs, files in os.walk(self.walkingDir): for theFile in files: currentDate = None currentSymbol = '' count = 0 # 确定合约【由文件夹名称决定,因为主力合约只能在文件夹名称中确定】 if '\\' in root: currentSymbol = root.split('\\')[-1] currentSymbol = currentSymbol # 排除不合法文件 if theFile.startswith('.'): continue # 确定日期 if '.' in theFile: pos = theFile.index('.') dateStr = theFile[:pos] currentDate = datetime.datetime.strptime(dateStr, '%Y%m%d') # 合约不存在 if not currentSymbol: print('*' * 6) print(u'没有找到合约') print('*' * 6) exit(0) # 检查是否有数据库记录 if currentDate and (not self.needUpdateChecking( currentSymbol, currentDate)): continue if '.csv' in theFile: # 读取文件 filePath = root + '\\' + theFile with open(filePath, 'r') as f: reader = csv.DictReader(f) # 开始导入数据 startTime = time() for row in reader: count += 1 if count == 1: print('=' * 6, currentSymbol, ' ', currentDate.date(), '=' * 6) # 过滤无效数据 timeStr = row['time'] theLen = len(timeStr) if theLen < 9: sub = 9 - theLen timeStr = '0' * sub + timeStr ''' hour = int(timeStr[0:2]) minute = int(timeStr[2:4]) second = int(timeStr[4:6]) microSecond = int(timeStr[6:9]) if hour < 0 or hour > 59 or minute < 0 or minute > 59 or second < 0 or second > 59 or microSecond < 0 or microSecond > 999: print u'删除时间格式错误数据:%s' % timeStr continue ''' try: tickTime = datetime.datetime.strptime( timeStr, '%H%M%S%f') except Exception: print(u'时间转换异常:%s' % timeStr) raise exit(0) t = tickTime.time() fakeData = True if self.dataType == 1: # 商品期货 if self.night == 0: # 无夜盘交易 if ((MORNING_START_CF <= t < MORNING_REST_CF) or (MORNING_RESTART_CF <= t < MORNING_END_CF) or (AFTERNOON_START_CF <= t < AFTERNOON_END_CF)): fakeData = False elif self.night == 1: # 夜间收盘 if ((MORNING_START_CF <= t < MORNING_REST_CF) or (MORNING_RESTART_CF <= t < MORNING_END_CF) or (AFTERNOON_START_CF <= t < AFTERNOON_END_CF) or (NIGHT_START_CF <= t < NIGHT_END_CF_N)): fakeData = False elif self.night == 2: # 凌晨收盘 if ((MORNING_START_CF <= t < MORNING_REST_CF) or (MORNING_RESTART_CF <= t < MORNING_END_CF) or (AFTERNOON_START_CF <= t < AFTERNOON_END_CF) or (NIGHT_START_CF <= t) or (NIGHT_END_CF_M > t)): fakeData = False elif self.dataType == 2: # 股指期货 if ((MORNING_START_SF <= t < MORNING_END_SF) or (AFTERNOON_START_SF <= t < AFTERNOON_END_SF)): fakeData = False theDate = row['calendarday'] theTime = tickTime.strftime('%H:%M:%S.%f') theDatetime = datetime.datetime.strptime( theDate + ' ' + theTime, '%Y%m%d %H:%M:%S.%f') # 数据库collection collection = self.db[currentSymbol] collection.create_index('datetime') if fakeData: print(t) print(u'【剔除无效数据】') # 删除已保存的数据库数据 flt = {'datetime': theDatetime} cursor = collection.find(flt) if cursor.count(): print(u'【剔除无效‘数据库’数据】') collection.delete_many(flt) continue # 创建VtTickData对象 tick = VtTickData() tick.vtSymbol = row['instrument'] tick.lastPrice = float(row['lastp']) tick.volume = int(row['volume']) tick.openInterest = int(row['openinterest']) tick.askPrice1 = float(row['ask1']) tick.askVolume1 = int(row['asksz1']) tick.bidPrice1 = float(row['bid1']) tick.bidVolume1 = int(row['bidsz1']) tick.date = theDate tick.time = theTime tick.datetime = theDatetime tick.tradingDay = row['tradingday'] # 保存tick到数据库 collection.update_many({'datetime': tick.datetime}, {'$set': tick.__dict__}, upsert=True) # 打印进程 if count: sub = time() - startTime print(u'用时:', sub, 's') print(u'数据量:', count, '\n') # 保存记录 if currentDate and count: self.updateHistory(currentSymbol, currentDate) print(u'所有数据导入完成')
def add_gap_ticks(self): """ 补充缺失的分时数据 :return: """ ds = DataSource() for vtSymbol in self.renkoDict.keys(): renkobar_list = self.renkoDict.get(vtSymbol, []) cache_bars = None cache_start_date = None cache_end_date = None for renko_bar in renkobar_list: # 通过mongo获取最新一个Renko bar的数据日期close时间 last_renko_dt = self.get_last_datetime(renko_bar.name) # 根据日期+vtSymbol,像datasource获取分钟数据,以close价格,转化为tick,推送到renko_bar中 if last_renko_dt is not None: start_date = last_renko_dt.strftime('%Y-%m-%d') else: start_date = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%d') end_date = (datetime.now() + timedelta(days=5)).strftime('%Y-%m-%d') self.writeDrLog(u'从datasource获取{}数据,开始日期:{}'.format( vtSymbol, start_date)) if cache_bars is None or cache_start_date != start_date or cache_end_date != end_date: fields = [ 'open', 'close', 'high', 'low', 'volume', 'open_interest', 'limit_up', 'limit_down', 'trading_date' ] cache_bars = ds.get_price(order_book_id=vtSymbol, start_date=start_date, end_date=end_date, frequency='1m', fields=fields) cache_start_date = start_date cache_end_date = end_date if cache_bars is not None: total = len(cache_bars) self.writeDrLog(u'一共获取{}条{} 1分钟数据'.format( total, vtSymbol)) if cache_bars is not None: self.writeDrLog(u'推送分时数据tick:{}到:{}'.format( vtSymbol, renko_bar.name)) for idx in cache_bars.index: row = cache_bars.loc[idx] tick = VtTickData() tick.vtSymbol = vtSymbol tick.symbol = vtSymbol last_bar_dt = datetime.strptime( str(idx), '%Y-%m-%d %H:%M:00') tick.datetime = last_bar_dt - timedelta(minutes=1) tick.date = tick.datetime.strftime('%Y-%m-%d') tick.time = tick.datetime.strftime('%H:%M:00') if tick.datetime.hour >= 21: if tick.datetime.isoweekday() == 5: # 星期五=》星期一 tick.tradingDay = ( tick.datetime + timedelta(days=3)).strftime('%Y-%m-%d') else: # 第二天 tick.tradingDay = ( tick.datetime + timedelta(days=1)).strftime('%Y-%m-%d') elif tick.datetime.hour < 8 and tick.datetime.isoweekday( ) == 6: # 星期六=>星期一 tick.tradingDay = ( tick.datetime + timedelta(days=2)).strftime('%Y-%m-%d') else: tick.tradingDay = tick.date tick.upperLimit = float(row['limit_up']) tick.lowerLimit = float(row['limit_down']) tick.lastPrice = float(row['close']) tick.askPrice1 = float(row['close']) tick.bidPrice1 = float(row['close']) tick.volume = int(row['volume']) tick.askVolume1 = tick.volume tick.bidVolume1 = tick.volume if last_renko_dt is not None and tick.datetime <= last_renko_dt: continue renko_bar.onTick(tick)