Esempio n. 1
0
    def processLine(self, line):
        historyData = line.split(',')
        #historyDataLen = len(historyData)
        symbol = historyData[2]
        #print 'processLine, symbol:' + symbol

        #从list转化为tick对象
        historytick = VtTickData()
        historytick._id = historyData[0]
        historytick.gatewayName = 'CTP'
        historytick.symbol = symbol
        historytick.TradingDay = historyData[1]
        historytick.exchange = historyData[3]
        historytick.vtSymbol = historytick.symbol  # '.'.join([tick.symbol, tick.exchange])

        historytick.lastPrice = self.convertFloatZero(historyData[5])
        #lastVolume
        historytick.volume = historyData[12]
        historytick.openInterest = historyData[14]

        UpdateMillisec = int(historyData[20])
        historytick.time = '.'.join([historyData[19], str(UpdateMillisec/ 100)])
        historytick.date = historyData[42]
        historytick.datetime = datetime.strptime(' '.join([historytick.date, historytick.time]), '%Y%m%d %H:%M:%S.%f')

        historytick.openPrice = self.convertFloatZero(historyData[9])
        historytick.highPrice = self.convertFloatZero(historyData[10])
        historytick.lowPrice = self.convertFloatZero(historyData[11])
        historytick.preClosePrice = self.convertFloatZero(historyData[12])

        historytick.ClosePrice = self.convertFloatZero(historyData[15])
        historytick.SettlementPrice = self.convertFloatZero(historyData[16])
        historytick.upperLimit = self.convertFloatZero(historyData[17])
        historytick.lowerLimit = self.convertFloatZero(historyData[18])

        # CTP只有一档行情
        historytick.bidPrice1 = self.convertFloatZero(historyData[21])
        historytick.bidVolume1 = historyData[22]
        historytick.askPrice1 = self.convertFloatZero(historyData[23])
        historytick.askVolume1 = historyData[24]

        historytick.AveragePrice = self.convertFloatZero(historyData[41])

        return historytick
Esempio n. 2
0
def loadqlCsv(fileName, dbName, symbol):
    """将TB极速版导出的csv格式的历史分钟数据插入到Mongo数据库中"""
    import csv
    count = 0
    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)

    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'],
                                 globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        tick = VtTickData()
        tick.vtSymbol = symbol
        tick.symbol = symbol
        tick.lastPrice = float(d[5])
        tick.volume = int(d[6])
        tick.openPrice = float(d[9])
        tick.highPrice = float(d[7])
        tick.lowPrice = float(d[8])
        tick.preClosePrice = float(d[20])
        tick.askPrice1 = float(d[12])
        tick.bidPrice1 = float(d[14])
        tick.askVolume1 = float(d[13])
        tick.bidVolume1 = float(d[15])
        count += 1

        tick.date = datetime.strptime(d[2], '%Y%m%d').strftime('%Y%m%d')

        if d[4] == '500':
            tick.time = d[3] + "." + "500000"
        else:
            tick.time = d[3] + "." + "000000"
        tick.datetime = datetime.strptime(tick.date + ' ' + tick.time,
                                          '%Y%m%d %H:%M:%S.%f')
        flt = {'datetime': tick.datetime}
        collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)

    print u'插入完毕,耗时:%s插入数量' % (time() - start), count
Esempio n. 3
0
def generateVtTick(row, symbol):
    """生成K线"""
    tick = VtTickData()
    tick.symbol = symbol
    tick.vtSymbol = symbol

    tick.lastPrice = row['last']
    tick.volume = row['volume']
    tick.openInterest = row['open_interest']
    tick.datetime = row.name
    tick.openPrice = row['open']
    tick.highPrice = row['high']
    tick.lowPrice = row['low']
    tick.preClosePrice = row['prev_close']
    tick.upperLimit = row['limit_up']
    tick.lowerLimit = row['limit_down']

    tick.bidPrice1 = row['b1']
    tick.bidPrice2 = row['b2']
    tick.bidPrice3 = row['b3']
    tick.bidPrice4 = row['b4']
    tick.bidPrice5 = row['b5']

    tick.bidVolume1 = row['b1_v']
    tick.bidVolume2 = row['b2_v']
    tick.bidVolume3 = row['b3_v']
    tick.bidVolume4 = row['b4_v']
    tick.bidVolume5 = row['b5_v']

    tick.askPrice1 = row['a1']
    tick.askPrice2 = row['a2']
    tick.askPrice3 = row['a3']
    tick.askPrice4 = row['a4']
    tick.askPrice5 = row['a5']

    tick.askVolume1 = row['a1_v']
    tick.askVolume2 = row['a2_v']
    tick.askVolume3 = row['a3_v']
    tick.askVolume4 = row['a4_v']
    tick.askVolume5 = row['a5_v']

    return tick
Esempio n. 4
0
def generateVtTick(row, symbol):
    """生成K线"""
    tick = VtTickData()
    tick.symbol = symbol
    tick.vtSymbol = symbol
    
    tick.lastPrice = row['last']
    tick.volume = row['volume']
    tick.openInterest = row['open_interest']
    tick.datetime = row.name
    tick.openPrice = row['open']
    tick.highPrice = row['high']
    tick.lowPrice = row['low']
    tick.preClosePrice = row['prev_close']
    tick.upperLimit = row['limit_up']
    tick.lowerLimit = row['limit_down']
    
    tick.bidPrice1 = row['b1']
    tick.bidPrice2 = row['b2']
    tick.bidPrice3 = row['b3']
    tick.bidPrice4 = row['b4']
    tick.bidPrice5 = row['b5']
    
    tick.bidVolume1 = row['b1_v']
    tick.bidVolume2 = row['b2_v']
    tick.bidVolume3 = row['b3_v']
    tick.bidVolume4 = row['b4_v']
    tick.bidVolume5 = row['b5_v']    
    
    tick.askPrice1 = row['a1']
    tick.askPrice2 = row['a2']
    tick.askPrice3 = row['a3']
    tick.askPrice4 = row['a4']
    tick.askPrice5 = row['a5']
    
    tick.askVolume1 = row['a1_v']
    tick.askVolume2 = row['a2_v']
    tick.askVolume3 = row['a3_v']
    tick.askVolume4 = row['a4_v']
    tick.askVolume5 = row['a5_v']        
    
    return tick
Esempio n. 5
0
    def startWork(self):
        for root, subdirs, files in os.walk(self.walkingDir):
            for theFile in files:
                currentDate = None
                currentSymbol = ''
                count = 0

                # 确定合约【由文件夹名称决定,因为主力合约只能在文件夹名称中确定】
                if '\\' in root:
                    currentSymbol = root.split('\\')[-1]
                    currentSymbol = currentSymbol

                # 排除不合法文件
                if theFile.startswith('.'):
                    continue

                # 确定日期
                if '.' in theFile:
                    pos = theFile.index('.')
                    dateStr = theFile[:pos]
                    currentDate = datetime.datetime.strptime(dateStr, '%Y%m%d')

                # 合约不存在
                if not currentSymbol:
                    print('*' * 6)
                    print(u'没有找到合约')
                    print('*' * 6)
                    exit(0)

                # 检查是否有数据库记录
                if currentDate and (not self.needUpdateChecking(
                        currentSymbol, currentDate)):
                    continue

                if '.csv' in theFile:
                    # 读取文件
                    filePath = root + '\\' + theFile
                    with open(filePath, 'r') as f:
                        reader = csv.DictReader(f)
                        # 开始导入数据
                        startTime = time()
                        for row in reader:
                            count += 1
                            if count == 1:
                                print('=' * 6, currentSymbol, ' ',
                                      currentDate.date(), '=' * 6)

                            # 过滤无效数据
                            timeStr = row['time']
                            theLen = len(timeStr)
                            if theLen < 9:
                                sub = 9 - theLen
                                timeStr = '0' * sub + timeStr
                            '''
                            hour = int(timeStr[0:2])
                            minute = int(timeStr[2:4])
                            second = int(timeStr[4:6])
                            microSecond = int(timeStr[6:9])
                            if hour < 0 or hour > 59 or minute < 0 or minute > 59 or second < 0 or second > 59 or microSecond < 0 or microSecond > 999:
                                print u'删除时间格式错误数据:%s' % timeStr
                                continue
                            '''

                            try:
                                tickTime = datetime.datetime.strptime(
                                    timeStr, '%H%M%S%f')
                            except Exception:
                                print(u'时间转换异常:%s' % timeStr)
                                raise
                                exit(0)

                            t = tickTime.time()
                            fakeData = True
                            if self.dataType == 1:
                                # 商品期货
                                if self.night == 0:
                                    # 无夜盘交易
                                    if ((MORNING_START_CF <= t <
                                         MORNING_REST_CF)
                                            or (MORNING_RESTART_CF <= t <
                                                MORNING_END_CF)
                                            or (AFTERNOON_START_CF <= t <
                                                AFTERNOON_END_CF)):
                                        fakeData = False
                                elif self.night == 1:
                                    # 夜间收盘
                                    if ((MORNING_START_CF <= t <
                                         MORNING_REST_CF)
                                            or (MORNING_RESTART_CF <= t <
                                                MORNING_END_CF)
                                            or (AFTERNOON_START_CF <= t <
                                                AFTERNOON_END_CF)
                                            or (NIGHT_START_CF <= t <
                                                NIGHT_END_CF_N)):
                                        fakeData = False
                                elif self.night == 2:
                                    # 凌晨收盘
                                    if ((MORNING_START_CF <= t <
                                         MORNING_REST_CF)
                                            or (MORNING_RESTART_CF <= t <
                                                MORNING_END_CF)
                                            or (AFTERNOON_START_CF <= t <
                                                AFTERNOON_END_CF)
                                            or (NIGHT_START_CF <= t)
                                            or (NIGHT_END_CF_M > t)):
                                        fakeData = False

                            elif self.dataType == 2:
                                # 股指期货
                                if ((MORNING_START_SF <= t < MORNING_END_SF)
                                        or (AFTERNOON_START_SF <= t <
                                            AFTERNOON_END_SF)):
                                    fakeData = False

                            theDate = row['calendarday']
                            theTime = tickTime.strftime('%H:%M:%S.%f')
                            theDatetime = datetime.datetime.strptime(
                                theDate + ' ' + theTime, '%Y%m%d %H:%M:%S.%f')

                            # 数据库collection
                            collection = self.db[currentSymbol]
                            collection.create_index('datetime')
                            if fakeData:
                                print(t)
                                print(u'【剔除无效数据】')
                                # 删除已保存的数据库数据
                                flt = {'datetime': theDatetime}
                                cursor = collection.find(flt)
                                if cursor.count():
                                    print(u'【剔除无效‘数据库’数据】')
                                collection.delete_many(flt)
                                continue

                            # 创建VtTickData对象
                            tick = VtTickData()
                            tick.vtSymbol = row['instrument']
                            tick.lastPrice = float(row['lastp'])
                            tick.volume = int(row['volume'])
                            tick.openInterest = int(row['openinterest'])
                            tick.askPrice1 = float(row['ask1'])
                            tick.askVolume1 = int(row['asksz1'])
                            tick.bidPrice1 = float(row['bid1'])
                            tick.bidVolume1 = int(row['bidsz1'])
                            tick.date = theDate
                            tick.time = theTime
                            tick.datetime = theDatetime
                            tick.tradingDay = row['tradingday']
                            # 保存tick到数据库
                            collection.update_many({'datetime': tick.datetime},
                                                   {'$set': tick.__dict__},
                                                   upsert=True)
                        # 打印进程
                        if count:
                            sub = time() - startTime
                            print(u'用时:', sub, 's')
                            print(u'数据量:', count, '\n')
                        #  保存记录
                        if currentDate and count:
                            self.updateHistory(currentSymbol, currentDate)
        print(u'所有数据导入完成')
Esempio n. 6
0
    def add_gap_ticks(self):
        """
        补充缺失的分时数据
        :return:
        """
        ds = DataSource()
        for vtSymbol in self.renkoDict.keys():
            renkobar_list = self.renkoDict.get(vtSymbol, [])
            cache_bars = None
            cache_start_date = None
            cache_end_date = None
            for renko_bar in renkobar_list:
                # 通过mongo获取最新一个Renko bar的数据日期close时间
                last_renko_dt = self.get_last_datetime(renko_bar.name)

                # 根据日期+vtSymbol,像datasource获取分钟数据,以close价格,转化为tick,推送到renko_bar中
                if last_renko_dt is not None:
                    start_date = last_renko_dt.strftime('%Y-%m-%d')
                else:
                    start_date = (datetime.now() -
                                  timedelta(days=90)).strftime('%Y-%m-%d')

                end_date = (datetime.now() +
                            timedelta(days=5)).strftime('%Y-%m-%d')
                self.writeDrLog(u'从datasource获取{}数据,开始日期:{}'.format(
                    vtSymbol, start_date))
                if cache_bars is None or cache_start_date != start_date or cache_end_date != end_date:
                    fields = [
                        'open', 'close', 'high', 'low', 'volume',
                        'open_interest', 'limit_up', 'limit_down',
                        'trading_date'
                    ]
                    cache_bars = ds.get_price(order_book_id=vtSymbol,
                                              start_date=start_date,
                                              end_date=end_date,
                                              frequency='1m',
                                              fields=fields)
                    cache_start_date = start_date
                    cache_end_date = end_date
                    if cache_bars is not None:
                        total = len(cache_bars)
                        self.writeDrLog(u'一共获取{}条{} 1分钟数据'.format(
                            total, vtSymbol))

                if cache_bars is not None:
                    self.writeDrLog(u'推送分时数据tick:{}到:{}'.format(
                        vtSymbol, renko_bar.name))
                    for idx in cache_bars.index:
                        row = cache_bars.loc[idx]
                        tick = VtTickData()
                        tick.vtSymbol = vtSymbol
                        tick.symbol = vtSymbol
                        last_bar_dt = datetime.strptime(
                            str(idx), '%Y-%m-%d %H:%M:00')
                        tick.datetime = last_bar_dt - timedelta(minutes=1)
                        tick.date = tick.datetime.strftime('%Y-%m-%d')
                        tick.time = tick.datetime.strftime('%H:%M:00')

                        if tick.datetime.hour >= 21:
                            if tick.datetime.isoweekday() == 5:
                                # 星期五=》星期一
                                tick.tradingDay = (
                                    tick.datetime +
                                    timedelta(days=3)).strftime('%Y-%m-%d')
                            else:
                                # 第二天
                                tick.tradingDay = (
                                    tick.datetime +
                                    timedelta(days=1)).strftime('%Y-%m-%d')
                        elif tick.datetime.hour < 8 and tick.datetime.isoweekday(
                        ) == 6:
                            # 星期六=>星期一
                            tick.tradingDay = (
                                tick.datetime +
                                timedelta(days=2)).strftime('%Y-%m-%d')
                        else:
                            tick.tradingDay = tick.date
                        tick.upperLimit = float(row['limit_up'])
                        tick.lowerLimit = float(row['limit_down'])
                        tick.lastPrice = float(row['close'])
                        tick.askPrice1 = float(row['close'])
                        tick.bidPrice1 = float(row['close'])
                        tick.volume = int(row['volume'])
                        tick.askVolume1 = tick.volume
                        tick.bidVolume1 = tick.volume

                        if last_renko_dt is not None and tick.datetime <= last_renko_dt:
                            continue
                        renko_bar.onTick(tick)