Пример #1
0
 def loadBar(self, dbName, collectionName, days):
     """从数据库中读取Bar数据,startDate是datetime对象"""
     startDate = self.today - timedelta(days)
     
     d = {'datetime':{'$gte':startDate}}
     barData = self.mainEngine.dbQuery(dbName, collectionName, d)
     
     l = []
     for d in barData:
         bar = VtBarData()
         bar.__dict__ = d
         l.append(bar)
     return l
Пример #2
0
    def loadBar(self, dbName, collectionName, days):
        """从数据库中读取Bar数据,startDate是datetime对象"""
        print "%s.%s.%s" % (__name__, self.__class__.__name__, get_current_function_name())
        startDate = self.today - timedelta(days)

        d = {'datetime': {'$gte': startDate}}
        barData = self.mainEngine.dbQuery(dbName, collectionName, d, 'datetime')

        l = []
        for d in barData:
            bar = VtBarData()
            bar.__dict__ = d
            l.append(bar)
        return l
Пример #3
0
 def loadBar(self, dbName, collectionName, days):
     """从数据库中读取Bar数据,startDate是datetime对象"""
     # 优先尝试从RQData获取数据
     if dbName == MINUTE_DB_NAME and collectionName.upper() in self.rqSymbolSet:
         l = self.loadRqBar(collectionName, days)
         return l
     
     # 如果没有则从数据库中读取数据
     startDate = self.today - timedelta(days)
     
     d = {'datetime':{'$gte':startDate}}
     barData = self.mainEngine.dbQuery(dbName, collectionName, d, 'datetime')
     
     l = []
     for d in barData:
         bar = VtBarData()
         bar.__dict__ = d
         l.append(bar)
     return l
Пример #4
0
    def loadData(self):
        """加载数据"""
        mc = MongoClient()
        db = mc[DAILY_DB_NAME]

        for vtSymbol in self.vtSymbolList:
            flt = {'datetime': {'$gte': self.startDt, '$lte': self.endDt}}

            collection = db[vtSymbol]
            cursor = collection.find(flt).sort('datetime')

            for d in cursor:
                bar = VtBarData()
                bar.__dict__ = d

                barDict = self.dataDict.setdefault(bar.datetime, OrderedDict())
                barDict[bar.vtSymbol] = bar

            self.output(u'%s数据加载完成,总数据量:%s' % (vtSymbol, cursor.count()))

        self.output(u'全部数据加载完成')
Пример #5
0
    def loadBar(self, dbName, collectionName, days):
        """从数据库中读取Bar数据,startDate是datetime对象"""
        # 优先尝试从RQData获取数据
        if dbName == MINUTE_DB_NAME and collectionName.upper(
        ) in self.rqSymbolSet:
            l = self.loadRqBar(collectionName, days)
            return l

        # 如果没有则从数据库中读取数据
        startDate = self.today - timedelta(days)

        d = {'datetime': {'$gte': startDate}}
        barData = self.mainEngine.dbQuery(dbName, collectionName, d,
                                          'datetime')

        l = []
        for d in barData:
            bar = VtBarData()
            bar.__dict__ = d
            l.append(bar)
        return l
Пример #6
0
 def new_bar_from_tick(self, tick, freq=None):
     bar = VtBarData()
     bar.vtSymbol = tick.vtSymbol
     bar.symbol = tick.symbol
     bar.exchange = tick.exchange
     bar.gatewayName = tick.gatewayName
     return self.override_bar_with_tick(bar, tick, freq=freq)
Пример #7
0
    def updateDay(self,  bar):
        newDay = False

        # 尚未创建对象
        if not self.dayBar:
            self.dayBar = VtBarData()
            newDay = True
        elif self.dayBar.TradingDay != bar.TradingDay:
            print "updateDay:%s" % bar.date
            # 生成上一X分钟K线的时间戳
            self.dayBar.datetime = self.dayBar.datetime.replace(hour=0,minute=0,second=0, microsecond=0)  # 将秒和微秒设为0
            self.dayBar.date = self.dayBar.datetime.strftime('%Y%m%d')
            self.dayBar.time = self.dayBar.datetime.strftime('%H:%M:%S.%f')

            # 推送
            self.onDayBar(self.dayBar)

            # 清空老K线缓存对象
            self.dayBar = VtBarData()
            newDay = True

        if newDay:
            self.dayBar.vtSymbol = bar.vtSymbol
            self.dayBar.symbol = bar.symbol
            self.dayBar.exchange = bar.exchange
            self.dayBar.TradingDay = bar.TradingDay

            self.dayBar.open = bar.open
            self.dayBar.high = bar.high
            self.dayBar.low = bar.low
        else:
            # 累加老K线
            self.dayBar.high = max(self.dayBar.high, bar.high)
            self.dayBar.low = min(self.dayBar.low, bar.low)

        # 通用部分
        self.dayBar.close = bar.close
        self.dayBar.datetime = bar.datetime
        self.dayBar.openInterest = bar.openInterest
        self.dayBar.volume += int(bar.volume)
Пример #8
0
 def addSymbolFreq(self, symbol, freq, size = 50):
     key = symbol + '_' + freq
     self.symbolFreqData[key] = ArrayManager(size=size)
     data_list=[]
     if freq == '1MIN':
         data_list = self.mainEngine.dbQuery( MINUTE_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
     elif freq == '5MIN':
         data_list = self.mainEngine.dbQuery( MINUTE_5_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
     elif freq == '15MIN':
         data_list = self.mainEngine.dbQuery( MINUTE_15_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
     elif freq == '30MIN':
         data_list = self.mainEngine.dbQuery( MINUTE_30_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
     elif freq == '60MIN':
         data_list = self.mainEngine.dbQuery( MINUTE_60_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
     elif freq == 'D':
         data_list = self.mainEngine.dbQuery( DAILY_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
     elif freq == 'W':
         data_list = self.mainEngine.dbQuery( WEEKLY_DB_NAME, symbol, d={}, sortKey='datetime', sortDirection=ASCENDING)
                                    
     if np.size(data_list) >= size:
         data_list = data_list[-1*size:]
         
     for data in data_list:
         bar = VtBarData()
         bar.close = data['close']
         bar.open = data['open']
         bar.high = data['high']
         bar.low = data['low']
         self.symbolFreqData[key].updateBar(bar)
Пример #9
0
 def new_bar_from_bar(self, bar, freq=None):
     bar2 = VtBarData()
     bar2.vtSymbol = bar.vtSymbol
     bar2.symbol = bar.symbol
     bar2.exchange = bar.exchange
     bar2.gatewayName = bar.gatewayName
     return self.override_bar_with_bar(bar2, bar, freq=freq)
Пример #10
0
 def loadData(self):
     """加载数据"""
     mc = MongoClient()
     db = mc[DAILY_DB_NAME]
     
     for vtSymbol in self.vtSymbolList:
         flt = {'datetime':{'$gte':self.startDt,
                            '$lte':self.endDt}} 
         
         collection = db[vtSymbol]
         cursor = collection.find(flt).sort('datetime')
         
         for d in cursor:
             bar = VtBarData()
             bar.__dict__ = d
             
             barDict = self.dataDict.setdefault(bar.datetime, OrderedDict())
             barDict[bar.vtSymbol] = bar
         
         self.output(u'%s数据加载完成,总数据量:%s' %(vtSymbol, cursor.count()))
     
     self.output(u'全部数据加载完成')
Пример #11
0
 def onTick(self, tick):
     """收到行情TICK推送(必须由用户继承实现)"""
     bar = VtBarData()
     bar.open = tick.lastPrice
     bar.close = tick.lastPrice
     bar.low = tick.lastPrice
     bar.high = tick.lastPrice
     self.onBar(bar)
Пример #12
0
def loadCsv(filename):
    """"""
    symbol = filename.split('.')[0]

    mc = MongoClient()
    db = mc[DAILY_DB_NAME]
    collection = db[symbol]

    with open(filename) as f:
        r = DictReader(f)
        for d in r:
            bar = VtBarData()
            bar.datetime = datetime.strptime(d['date'], '%Y/%m/%d')
            bar.vtSymbol = symbol
            bar.open = float(d['open'])
            bar.high = float(d['high'])
            bar.low = float(d['low'])
            bar.close = float(d['close'])
            bar.volume = int(d['volume'])

            collection.insert(bar.__dict__)
Пример #13
0
    def updateWCandle(self, Candle):
        """周K线更新"""
        # 尚未创建对象
        abstract_week = Candle.datetime.strftime('%W')
        if not self.intraWeek:
            self.intraWeek = abstract_week
        if abstract_week != self.intraWeek:
            # 推送
            if self.WeekCandle:
                self.onWCandle(self.WeekCandle)
                # 清空老K线缓存对象
                self.WeekCandle = None

        if not self.WeekCandle:
            self.WeekCandle = VtBarData()

            self.WeekCandle.vtSymbol = Candle.vtSymbol
            self.WeekCandle.symbol = Candle.symbol
            self.WeekCandle.exchange = Candle.exchange

            self.WeekCandle.open = Candle.open
            self.WeekCandle.high = Candle.high
            self.WeekCandle.low = Candle.low
            self.WeekCandle.datetime = Candle.datetime.replace(
                second=0, microsecond=0)  # 将秒和微秒设为0
            self.WeekCandle.date = Candle.datetime.strftime('%Y%m%d')
            self.WeekCandle.time = Candle.datetime.strftime('%H:%M:%S.%f')

        # 累加老K线
        self.WeekCandle.high = max(self.WeekCandle.high, Candle.high)
        self.WeekCandle.low = min(self.WeekCandle.low, Candle.low)
        self.WeekCandle.close = Candle.close
        self.WeekCandle.openInterest = Candle.openInterest
        self.WeekCandle.volume += Candle.volume
        self.intraWeek = abstract_week

        if (bar.datetime.hour, bar.datetime.minute
            ) == self.marketClose and self.marketClose != (23, 59):
            if Candle.datetime.strftime('%w') == 5:  # 每周五收盘强切周线
                self.onWCandle(self.WeekCandle)
                self.WeekCandle = None
        elif (bar.datetime.hour, bar.datetime.minute
              ) == self.marketClose and self.marketClose == (23, 59):
            if Candle.datetime.strftime('%w') == 0:  # 7*24市场在周日晚0点切
                self.onWCandle(self.WeekCandle)
                self.WeekCandle = None
Пример #14
0
    def updateBar(self, bar):
        """1分钟K线更新"""
        # 尚未创建对象
        if not self.xminBar:
            self.xminBar = VtBarData()

            self.xminBar.vtSymbol = bar.vtSymbol
            self.xminBar.symbol = bar.symbol
            self.xminBar.exchange = bar.exchange

            self.xminBar.open = bar.open
            self.xminBar.high = bar.high
            self.xminBar.low = bar.low

            self.xminBar.datetime = bar.datetime  # 以第一根分钟K线的开始时间戳作为X分钟线的时间戳
        # 累加老K线
        else:
            self.xminBar.high = max(self.xminBar.high, bar.high)
            self.xminBar.low = min(self.xminBar.low, bar.low)

        # 通用部分
        self.xminBar.close = bar.close
        self.xminBar.openInterest = bar.openInterest
        self.xminBar.volume += int(bar.volume)

        # X分钟已经走完
        if not (bar.datetime.minute + 1) % self.xmin:  # 可以用X整除
            # 生成上一X分钟K线的时间戳
            self.xminBar.datetime = self.xminBar.datetime.replace(
                second=0, microsecond=0)  # 将秒和微秒设为0
            self.xminBar.date = self.xminBar.datetime.strftime('%Y%m%d')
            self.xminBar.time = self.xminBar.datetime.strftime('%H:%M:%S.%f')

            # 推送
            if self.xminBar.low == 0 or self.xminBar.high == 0 or self.xminBar.low == self.xminBar.high:
                return
            else:
                self.onXminBar(self.xminBar, self.lastTick)

            # 清空老K线缓存对象
            self.xminBar = None

        # 更新交易日
        self.tradingDay
Пример #15
0
def loadCsv(filename):
    """"""
    symbol = filename.split('.')[0]
    
    mc = MongoClient()
    db = mc[DAILY_DB_NAME]
    collection = db[symbol]
    
    with open(filename) as f:
        r = DictReader(f)
        for d in r:
            bar = VtBarData()
            bar.datetime = datetime.strptime(d['date'], '%Y/%m/%d')
            bar.vtSymbol = symbol
            bar.open = float(d['open'])
            bar.high = float(d['high'])
            bar.low = float(d['low'])
            bar.close = float(d['close'])
            bar.volume= int(d['volume'])
        
            collection.insert(bar.__dict__)
Пример #16
0
    def updateBar(self, bar):
        """1分钟K线更新"""
        # 尚未创建对象
        if not self.xminBar:
            self.xminBar = VtBarData()

            self.xminBar.vtSymbol = bar.vtSymbol
            self.xminBar.symbol = bar.symbol
            self.xminBar.exchange = bar.exchange

            self.xminBar.open = bar.open
            self.xminBar.high = bar.high
            self.xminBar.low = bar.low
            # 生成上一X分钟K线的时间戳
            self.xminBar.datetime = bar.datetime.replace(
                second=0, microsecond=0)  # 将秒和微秒设为0
            self.xminBar.date = bar.datetime.strftime('%Y%m%d')
            self.xminBar.time = bar.datetime.strftime('%H:%M:%S.%f')

            # 累加老K线
        else:
            self.xminBar.high = max(self.xminBar.high, bar.high)
            self.xminBar.low = min(self.xminBar.low, bar.low)

        # 通用部分
        self.xminBar.close = bar.close
        # self.xminBar.datetime = bar.datetime
        self.xminBar.openInterest = bar.openInterest
        self.xminBar.volume += int(bar.volume)

        # X分钟已经走完
        if self.xmin < 61:
            if not (bar.datetime.minute + 1) % self.xmin:  # 可以用X整除
                # 推送
                self.onXminBar(self.xminBar)
                # 清空老K线缓存对象
                self.xminBar = None
        elif self.xmin > 60:
            if not (bar.datetime.hour *
                    60) % self.xmin and bar.datetime.minute == 0:  # 小时线
                self.onXminBar(self.xminBar)
                # 清空老K线缓存对象
                self.xminBar = None
Пример #17
0
    def updateBar(self, bar):
        """1分钟K线更新"""

        # 尚未创建对象
        if not self.xminBar:
            self.xminBar = VtBarData()

            self.xminBar.vtSymbol = bar.vtSymbol
            self.xminBar.symbol = bar.symbol
            self.xminBar.exchange = bar.exchange

            self.xminBar.open = bar.open
            self.xminBar.high = bar.high
            self.xminBar.low = bar.low
        # 累加老K线
        else:
            self.xminBar.high = max(self.xminBar.high, bar.high)
            self.xminBar.low = min(self.xminBar.low, bar.low)

        # 通用部分
        self.xminBar.close = bar.close
        self.xminBar.datetime = bar.datetime
        self.xminBar.openInterest = bar.openInterest
        self.xminBar.volume += int(bar.volume)

        # X分钟已经走完
        # if not bar.datetime.minute % self.xmin:   # 可以用X整除
        if not (bar.datetime.minute +
                1) % self.xmin:  # 可以用X整除,原写法会造成早上开盘第一根bar就算合成了一根bar,张英杰修改
            # 生成上一X分钟K线的时间戳
            self.xminBar.datetime = self.xminBar.datetime.replace(
                second=0, microsecond=0)  # 将秒和微秒设为0
            self.xminBar.date = self.xminBar.datetime.strftime('%Y%m%d')
            self.xminBar.time = self.xminBar.datetime.strftime('%H:%M:%S.%f')

            # 推送

            self.onXminBar(self.xminBar)

            # 清空老K线缓存对象
            self.xminBar = None
Пример #18
0
    def updateMyBar(self, minute, bar):
        """1分钟K线更新"""
        # 尚未创建对象
        if not self.myXminBar[minute]:
            self.myXminBar[minute] = VtBarData()

            self.myXminBar[minute].vtSymbol = bar.vtSymbol
            self.myXminBar[minute].symbol = bar.symbol
            self.myXminBar[minute].exchange = bar.exchange

            self.myXminBar[minute].open = bar.open
            self.myXminBar[minute].high = bar.high
            self.myXminBar[minute].low = bar.low
        # 累加老K线
        else:
            self.myXminBar[minute].high = max(self.myXminBar[minute].high,
                                              bar.high)
            self.myXminBar[minute].low = min(self.myXminBar[minute].low,
                                             bar.low)

        # 通用部分
        self.myXminBar[minute].close = bar.close
        self.myXminBar[minute].datetime = bar.datetime
        self.myXminBar[minute].openInterest = bar.openInterest
        self.myXminBar[minute].volume += int(bar.volume)

        # X分钟已经走完
        if (not bar.datetime.minute % minute):  # 可以用X整除
            # 生成上一X分钟K线的时间戳
            self.myXminBar[minute].datetime = self.myXminBar[
                minute].datetime.replace(second=0, microsecond=0)  # 将秒和微秒设为0
            self.myXminBar[minute].date = self.myXminBar[
                minute].datetime.strftime('%Y%m%d')
            self.myXminBar[minute].time = self.myXminBar[
                minute].datetime.strftime('%H:%M:%S.%f')

            # 推送
            self.onXBar(minute, self.myXminBar[minute])

            # 清空老K线缓存对象
            self.myXminBar[minute] = None
Пример #19
0
    def updateDay(self, bar):
        """天更新"""
        # 尚未创建对象
        if not self.dayBar:
            self.dayBar = VtBarData()

            self.dayBar.vtSymbol = bar.vtSymbol
            self.dayBar.symbol = bar.symbol
            self.dayBar.exchange = bar.exchange

            self.dayBar.open = bar.open
            self.dayBar.high = bar.high
            self.dayBar.low = bar.low
        # 累加老K线
        else:
            self.dayBar.high = max(self.dayBar.high, bar.high)
            self.dayBar.low = min(self.dayBar.low, bar.low)

        # 通用部分
        self.dayBar.close = bar.close
        self.dayBar.datetime = bar.datetime
        self.dayBar.openInterest = bar.openInterest
        self.dayBar.volume += int(bar.volume)

        # 判断天数
        if bar.datetime.time() == self.DAY_END:  #
            print "updateDay:%s" % bar.date
            # 生成上一X分钟K线的时间戳
            self.dayBar.datetime = self.dayBar.datetime.replace(
                hour=0, minute=0, second=0, microsecond=0)  # 将秒和微秒设为0
            self.dayBar.date = self.dayBar.datetime.strftime('%Y%m%d')
            self.dayBar.time = self.dayBar.datetime.strftime('%H:%M:%S.%f')

            # 推送
            self.onDayBar(self.dayBar)

            # 清空老K线缓存对象
            self.dayBar = None
Пример #20
0
    def updateGivingBar(self, bar):
        #givingBar这个变量只在本函数内使用
        if not self.givingBar:
            self.givingBar = VtBarData()

            self.givingBar.vtSymbol = bar.vtSymbol
            self.givingBar.symbol = bar.symbol
            self.givingBar.exchange = bar.exchange
            self.givingBar.TradingDay = bar.TradingDay

            self.givingBar.open = bar.open
            self.givingBar.high = bar.high
            self.givingBar.low = bar.low

            self.givingBar.datetime = bar.datetime
        else:
            self.givingBar.high = max(self.givingBar.high, bar.high)
            self.givingBar.low = min(self.givingBar.low, bar.low)

        # 通用部分
        self.givingBar.close = bar.close
        self.givingBar.openInterest = bar.openInterest
        self.givingBar.volume += int(bar.volume)
Пример #21
0
    def __firstTick(self, tick):
        """ K线的第一个Tick数据"""
        self.bar = VtBarData()  # 创建新的K线

        self.bar.vtSymbol = tick.vtSymbol
        self.bar.symbol = tick.symbol
        self.bar.exchange = tick.exchange

        self.bar.open = tick.lastPrice  # O L H C
        self.bar.high = tick.lastPrice
        self.bar.low = tick.lastPrice
        self.bar.close = tick.lastPrice

        # K线的日期时间
        self.bar.date = tick.date  # K线的日期时间(去除秒)设为第一个Tick的时间
        self.bar.time = tick.time  # K线的日期时间(去除秒)设为第一个Tick的时间
        self.bar.datetime = tick.datetime

        self.bar.volume = tick.volume
        self.bar.openInterest = tick.openInterest

        self.barFirstTick = True  # 标识该Tick属于该Bar的第一个tick数据

        self.lineBar.append(self.bar)  # 推入到lineBar队列
Пример #22
0
    def updateMCandle(self, Candle):
        """月K线更新"""
        # 尚未创建对象
        abstract_month = int(Candle.datetime.strftime('%m'))
        if not self.intraMonth:
            self.intraMonth = abstract_month

        if abstract_month != self.intraMonth:
            # 推送
            if self.MonthCandle:
                self.onMCandle(self.MonthCandle)
                # 清空老K线缓存对象
                self.MonthCandle = None

        if not self.MonthCandle:
            self.MonthCandle = VtBarData()

            self.MonthCandle.vtSymbol = Candle.vtSymbol
            self.MonthCandle.symbol = Candle.symbol
            self.MonthCandle.exchange = Candle.exchange

            self.MonthCandle.open = Candle.open
            self.MonthCandle.high = Candle.high
            self.MonthCandle.low = Candle.low
            self.MonthCandle.datetime = Candle.datetime.replace(
                second=0, microsecond=0)  # 将秒和微秒设为0
            self.MonthCandle.date = Candle.datetime.strftime('%Y%m%d')
            self.MonthCandle.time = Candle.datetime.strftime('%H:%M:%S.%f')

        # 累加老K线
        self.MonthCandle.high = max(self.MonthCandle.high, Candle.high)
        self.MonthCandle.low = min(self.MonthCandle.low, Candle.low)
        self.MonthCandle.close = Candle.close
        self.MonthCandle.openInterest = Candle.openInterest
        self.MonthCandle.volume += Candle.volume
        self.intraMonth = abstract_month
Пример #23
0
def loadTdxCsv(fileName, dbName, symbol):
    """将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
    import csv

    start = time()
    date_correct = ""
    print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))

    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'],
                                 globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        bar = VtBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d[1])
        bar.high = float(d[2])
        bar.low = float(d[3])
        bar.close = float(d[4])
        #通达信的夜盘时间按照新的一天计算,此处将其按照当天日期统计,方便后续查阅
        date_temp, time_temp = d[0].strip(' ').replace('\xef\xbb\xbf',
                                                       '').split('-', 1)
        if time_temp == '15:00':
            date_correct = date_temp
        if time_temp[:
                     2] == "21" or time_temp[:
                                             2] == "22" or time_temp[:
                                                                     2] == "23":
            date_temp = date_correct

        bar.date = datetime.strptime(date_temp, '%Y/%m/%d').strftime('%Y%m%d')
        bar.time = time_temp[:2] + ':' + time_temp[3:5] + ':00'
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
                                         '%Y%m%d %H:%M:%S')
        bar.volume = d[5]

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)

    print('插入完毕,耗时:%s' % (time() - start))
Пример #24
0
def generateVtBar(vtSymbol, d):
    """生成K线"""
    bar = VtBarData()
    bar.vtSymbol = vtSymbol
    bar.symbol, bar.exchange = bar.vtSymbol.split('.')
    
    bar.datetime = datetime.datetime.fromtimestamp(d['time'])
    bar.date = bar.datetime.strftime('%Y%m%d')
    bar.time = bar.datetime.strftime('%H:%M:%S')
    
    bar.open = d['open']
    bar.high = d['high']
    bar.low = d['low']
    bar.close = d['close']
    bar.volume = d['volumeto']
    
    return bar
Пример #25
0
def generateVtBar(symbol, d):
    """生成K线"""
    bar = VtBarData()

    bar.symbol = symbol
    bar.vtSymbol = symbol
    bar.open = d['open']
    bar.high = d['high']
    bar.low = d['low']
    bar.close = d['close']
    bar.volume = d['volume']
    bar.openInterest = d['open_oi']
    bar.datetime = datetime.fromtimestamp(d['datetime'] / 1000000000)
    bar.date = bar.datetime.strftime("%Y%m%d")
    bar.time = bar.datetime.strftime("%H:%M:%S")

    return bar
Пример #26
0
    def downloadFuturesDailyBar(self, symbol):
        """
        下载期货合约的日行情,symbol是合约代码,
        若最后四位为0000(如IF0000),代表下载连续合约。
        """
        print u'开始下载%s日行情' % symbol

        # 查询数据库中已有数据的最后日期
        cl = self.dbClient[DAILY_DB_NAME][symbol]
        cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
        if cx.count():
            last = cx[0]
        else:
            last = ''

        # 主力合约
        if '0000' in symbol:
            path = 'api/market/getMktMFutd.json'

            params = {}
            params['contractObject'] = symbol.replace('0000', '')
            params['mainCon'] = 1
            if last:
                params['startDate'] = last['date']
        # 交易合约
        else:
            path = 'api/market/getMktFutd.json'

            params = {}
            params['ticker'] = symbol
            if last:
                params['startDate'] = last['date']

        # 开始下载数据
        data = self.datayesClient.downloadData(path, params)

        if data:
            # 创建datetime索引
            self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
                [('datetime', pymongo.ASCENDING)], unique=True)

            for d in data:
                bar = VtBarData()
                bar.vtSymbol = symbol
                bar.symbol = symbol
                try:
                    bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
                        d.get('exchangeCD', ''), '')
                    bar.open = d.get('openPrice', 0)
                    bar.high = d.get('highestPrice', 0)
                    bar.low = d.get('lowestPrice', 0)
                    bar.close = d.get('closePrice', 0)
                    bar.date = d.get('tradeDate', '').replace('-', '')
                    bar.time = ''
                    bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
                    bar.volume = d.get('turnoverVol', 0)
                    bar.openInterest = d.get('openInt', 0)
                except KeyError:
                    print d

                flt = {'datetime': bar.datetime}
                self.dbClient[DAILY_DB_NAME][symbol].update_one(
                    flt, {'$set': bar.__dict__}, upsert=True)

                print u'%s下载完成' % symbol
        else:
            print u'找不到合约%s' % symbol
Пример #27
0
    def downloadEquityDailyBar(self, symbol):
        """
        下载股票的日行情,symbol是股票代码
        """
        print u'开始下载%s日行情' % symbol

        # 查询数据库中已有数据的最后日期
        cl = self.dbClient[DAILY_DB_NAME][symbol]
        cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
        if cx.count():
            last = cx[0]
        else:
            last = ''

        # 开始下载数据
        path = 'api/market/getMktEqud.json'

        params = {}
        params['ticker'] = symbol
        if last:
            params['beginDate'] = last['date']

        data = self.datayesClient.downloadData(path, params)

        if data:
            # 创建datetime索引
            self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
                [('datetime', pymongo.ASCENDING)], unique=True)

            for d in data:
                bar = VtBarData()
                bar.vtSymbol = symbol
                bar.symbol = symbol
                try:
                    bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
                        d.get('exchangeCD', ''), '')
                    bar.open = d.get('openPrice', 0)
                    bar.high = d.get('highestPrice', 0)
                    bar.low = d.get('lowestPrice', 0)
                    bar.close = d.get('closePrice', 0)
                    bar.date = d.get('tradeDate', '').replace('-', '')
                    bar.time = ''
                    bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
                    bar.volume = d.get('turnoverVol', 0)
                except KeyError:
                    print d

                flt = {'datetime': bar.datetime}
                self.dbClient[DAILY_DB_NAME][symbol].update_one(
                    flt, {'$set': bar.__dict__}, upsert=True)

            print u'%s下载完成' % symbol
        else:
            print u'找不到合约%s' % symbol
Пример #28
0
def generateVtBar(symbol, d):
    """生成K线"""
    l = symbol.split('_')
    bar = VtBarData()
    bar.symbol = l[-2] + l[-1]
    bar.exchange = l[0]
    bar.vtSymbol = '/'.join([bar.symbol, bar.exchange])
    bar.datetime = datetime.datetime.strptime(d['time_open'], '%Y-%m-%dT%H:%M:%S.%f0Z')
    bar.date = bar.datetime.strftime('%Y%m%d')
    bar.time = bar.datetime.strftime('%H:%M:%S')
    bar.open = d['price_open']
    bar.high = d['price_high']
    bar.low = d['price_low']
    bar.close = d['price_close']
    bar.volume = d['volume_traded']
    
    return bar
Пример #29
0
def loadTdxLc1(fileName, dbName, symbol):
    """将通达信导出的lc1格式的历史分钟数据插入到Mongo数据库中"""
    from struct import *

    start = time()

    print u'开始读取通达信Lc1文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
    
    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)  

    #读取二进制文件
    ofile=open(fileName,'rb')
    buf=ofile.read()
    ofile.close()
    
    num=len(buf)
    no=num/32
    b=0
    e=32  
    dl = []
    for i in xrange(no):
        a=unpack('hhfffffii',buf[b:e])
        b=b+32
        e=e+32
        bar = VtBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = a[2]
        bar.high = a[3]
        bar.low = a[4]
        bar.close = a[5]
        bar.date = str(int(a[0]/2048)+2004)+str(int(a[0]%2048/100)).zfill(2)+str(a[0]%2048%100).zfill(2)
        bar.time = str(int(a[1]/60)).zfill(2)+':'+str(a[1]%60).zfill(2)+':00'
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
        bar.volume = a[7]

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
    
    print u'插入完毕,耗时:%s' % (time()-start)
Пример #30
0
def loadTdxCsv(fileName, dbName, symbol):
    """将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
    import csv
    
    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
    
    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)   
    
    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        bar = VtBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d[2])
        bar.high = float(d[3])
        bar.low = float(d[4])
        bar.close = float(d[5])
        bar.date = datetime.strptime(d[0], '%Y/%m/%d').strftime('%Y%m%d')
        bar.time = d[1][:2]+':'+d[1][2:4]+':00'
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
        bar.volume = d[6]
        bar.openInterest = d[7]

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)  
        print bar.date, bar.time
    
    print u'插入完毕,耗时:%s' % (time()-start)
Пример #31
0
    def downloadFuturesDailyBar(self, symbol):
        """
        下载期货合约的日行情,symbol是合约代码,
        若最后四位为0000(如IF0000),代表下载连续合约。
        """
        print u'开始下载%s日行情' %symbol
        
        # 查询数据库中已有数据的最后日期
        cl = self.dbClient[DAILY_DB_NAME][symbol]
        cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
        if cx.count():
            last = cx[0]
        else:
            last = ''
        
        # 主力合约
        if '0000' in symbol:
            path = 'api/market/getMktMFutd.json'
            
            params = {}
            params['contractObject'] = symbol.replace('0000', '')
            params['mainCon'] = 1
            if last:
                params['startDate'] = last['date']
        # 交易合约
        else:
            path = 'api/market/getMktFutd.json'
            
            params = {}
            params['ticker'] = symbol
            if last:
                params['startDate'] = last['date']
        
        # 开始下载数据
        data = self.datayesClient.downloadData(path, params)
        
        if data:
            # 创建datetime索引
            self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)], 
                                                                      unique=True)                

            for d in data:
                bar = VtBarData()
                bar.vtSymbol = symbol
                bar.symbol = symbol
                try:
                    bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
                    bar.open = d.get('openPrice', 0)
                    bar.high = d.get('highestPrice', 0)
                    bar.low = d.get('lowestPrice', 0)
                    bar.close = d.get('closePrice', 0)
                    bar.date = d.get('tradeDate', '').replace('-', '')
                    bar.time = ''
                    bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
                    bar.volume = d.get('turnoverVol', 0)
                    bar.openInterest = d.get('openInt', 0)
                except KeyError:
                    print d
                
                flt = {'datetime': bar.datetime}
                self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)            
            
                print u'%s下载完成' %symbol
        else:
            print u'找不到合约%s' %symbol
Пример #32
0
    def onBar(self, bar):
        """收到Bar推送(必须由用户继承实现)"""
        # 如果当前是一个5分钟走完
        if bar.datetime.minute % 5 == 0:
            # 如果已经有聚合5分钟K线
            if self.min5Bar:
                # 将最新分钟的数据更新到目前5分钟线中
                min5Bar = self.min5Bar
                min5Bar.high = max(min5Bar.high, bar.high)
                min5Bar.low = min(min5Bar.low, bar.low)
                min5Bar.close = bar.close

                # 推送5分钟线数据
                self.onFiveBar(min5Bar)

                # 清空5分钟线数据缓存
                self.min5Bar = None
        else:
            # 如果没有缓存则新建
            if not self.min5Bar:
                min5Bar = VtBarData()

                min5Bar.vtSymbol = bar.vtSymbol
                min5Bar.symbol = bar.symbol
                min5Bar.exchange = bar.exchange

                min5Bar.open = bar.open
                min5Bar.high = bar.high
                min5Bar.low = bar.low
                min5Bar.close = bar.close

                min5Bar.date = bar.date
                min5Bar.time = bar.time
                min5Bar.datetime = bar.datetime

                self.min5Bar = min5Bar
            else:
                min5Bar = self.min5Bar
                min5Bar.high = max(min5Bar.high, bar.high)
                min5Bar.low = min(min5Bar.low, bar.low)
                min5Bar.close = bar.close

        # ----------------------------------------------------------------------

        if bar.datetime.minute % 15 == 0:
            # 如果已经有聚合15分钟K线
            if self.min15Bar:
                # 将最新分钟的数据更新到目前5分钟线中
                min15Bar = self.min15Bar
                min15Bar.high = max(min15Bar.high, bar.high)
                min15Bar.low = min(min15Bar.low, bar.low)
                min15Bar.close = bar.close

                # 推送5分钟线数据
                self.onFifteenBar(min15Bar)

                # 清空5分钟线数据缓存
                self.min15Bar = None
        else:
            # 如果没有缓存则新建
            if not self.min15Bar:
                min15Bar = VtBarData()

                min15Bar.vtSymbol = bar.vtSymbol
                min15Bar.symbol = bar.symbol
                min15Bar.exchange = bar.exchange

                min15Bar.open = bar.open
                min15Bar.high = bar.high
                min15Bar.low = bar.low
                min15Bar.close = bar.close

                min15Bar.date = bar.date
                min15Bar.time = bar.time
                min15Bar.datetime = bar.datetime

                self.min15Bar = min15Bar
            else:
                min15Bar = self.min15Bar
                min15Bar.high = max(min15Bar.high, bar.high)
                min15Bar.low = min(min15Bar.low, bar.low)
                min15Bar.close = bar.close

        # ----------------------------------------------------------------------
        # 如果当前是一个30分钟走完
        if bar.datetime.minute % 30 == 0:
            # 如果已经有聚合30分钟K线
            if self.min30Bar:
                # 将最新分钟的数据更新到目前5分钟线中
                min30Bar = self.min30Bar
                min30Bar.high = max(min30Bar.high, bar.high)
                min30Bar.low = min(min30Bar.low, bar.low)
                min30Bar.close = bar.close

                # 推送5分钟线数据
                self.onThirtyBar(min30Bar)

                # 清空5分钟线数据缓存
                self.min30Bar = None
        else:
            # 如果没有缓存则新建
            if not self.min30Bar:
                min30Bar = VtBarData()

                min30Bar.vtSymbol = bar.vtSymbol
                min30Bar.symbol = bar.symbol
                min30Bar.exchange = bar.exchange

                min30Bar.open = bar.open
                min30Bar.high = bar.high
                min30Bar.low = bar.low
                min30Bar.close = bar.close

                min30Bar.date = bar.date
                min30Bar.time = bar.time
                min30Bar.datetime = bar.datetime

                self.min30Bar = min30Bar
            else:
                min30Bar = self.min30Bar
                min30Bar.high = max(min30Bar.high, bar.high)
                min30Bar.low = min(min30Bar.low, bar.low)
                min30Bar.close = bar.close

        # ----------------------------------------------------------------------
        # 如果当前是一个60分钟走完
        if bar.datetime.minute % 60 == 0:
            # 如果已经有聚合60分钟K线
            if self.min60Bar:
                # 将最新分钟的数据更新到目前5分钟线中
                min60Bar = self.min60Bar
                min60Bar.high = max(min60Bar.high, bar.high)
                min60Bar.low = min(min60Bar.low, bar.low)
                min60Bar.close = bar.close

                # 推送5分钟线数据
                self.onSixtyBar(min60Bar)

                # 清空5分钟线数据缓存
                self.min60Bar = None
        else:
            # 如果没有缓存则新建
            if not self.min60Bar:
                min60Bar = VtBarData()

                min60Bar.vtSymbol = bar.vtSymbol
                min60Bar.symbol = bar.symbol
                min60Bar.exchange = bar.exchange

                min60Bar.open = bar.open
                min60Bar.high = bar.high
                min60Bar.low = bar.low
                min60Bar.close = bar.close

                min60Bar.date = bar.date
                min60Bar.time = bar.time
                min60Bar.datetime = bar.datetime

                self.min60Bar = min60Bar
            else:
                min60Bar = self.min60Bar
                min60Bar.high = max(min60Bar.high, bar.high)
                min60Bar.low = min(min60Bar.low, bar.low)
                min60Bar.close = bar.close
Пример #33
0
def generateVtBar(d):
    """生成K线"""
    bar = VtBarData()
    
    bar.symbol = d['symbol']
    bar.vtSymbol = d['symbol']
    bar.date = d['date']
    bar.time = ':'.join([d['time'][:2], d['time'][2:]])
    bar.open = d['open']
    bar.high = d['high']
    bar.low = d['low']
    bar.close = d['close']
    bar.volume = d['volume']
    bar.openInterest = d['openInterest']
    bar.datetime = datetime.datetime.strptime(' '.join([bar.date, bar.time]), '%Y%m%d %H:%M')    
    
    return bar
Пример #34
0
def downloadEquityDailyBarts(self, symbol):
    """
    下载股票的日行情,symbol是股票代码
    """
    print u'开始下载%s日行情' %symbol
    
    # 查询数据库中已有数据的最后日期
    cl = self.dbClient[DAILY_DB_NAME][symbol]
    cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
    if cx.count():
        last = cx[0]
    else:
        last = ''
    # 开始下载数据
    import tushare as ts
    
    if last:
        start = last['date'][:4]+'-'+last['date'][4:6]+'-'+last['date'][6:]
        
    data = ts.get_k_data(symbol,start)
    
    if not data.empty:
        # 创建datetime索引
        self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)], 
                                                            unique=True)                
        
        for index, d in data.iterrows():
            bar = VtBarData()
            bar.vtSymbol = symbol
            bar.symbol = symbol
            try:
                bar.open = d.get('open')
                bar.high = d.get('high')
                bar.low = d.get('low')
                bar.close = d.get('close')
                bar.date = d.get('date').replace('-', '')
                bar.time = ''
                bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
                bar.volume = d.get('volume')
            except KeyError:
                print d
            
            flt = {'datetime': bar.datetime}
            self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)            
        
        print u'%s下载完成' %symbol
    else:
        print u'找不到合约%s' %symbol
Пример #35
0
    def downloadEquityDailyBar(self, symbol):
        """
        下载股票的日行情,symbol是股票代码
        """
        print u'开始下载%s日行情' %symbol
        
        # 查询数据库中已有数据的最后日期
        cl = self.dbClient[DAILY_DB_NAME][symbol]
        cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
        if cx.count():
            last = cx[0]
        else:
            last = ''
        
        # 开始下载数据
        path = 'api/market/getMktEqud.json'
            
        params = {}
        params['ticker'] = symbol
        if last:
            params['beginDate'] = last['date']
        
        data = self.datayesClient.downloadData(path, params)
        
        if data:
            # 创建datetime索引
            self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)], 
                                                                unique=True)                

            for d in data:
                bar = VtBarData()
                bar.vtSymbol = symbol
                bar.symbol = symbol
                try:
                    bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
                    bar.open = d.get('openPrice', 0)
                    bar.high = d.get('highestPrice', 0)
                    bar.low = d.get('lowestPrice', 0)
                    bar.close = d.get('closePrice', 0)
                    bar.date = d.get('tradeDate', '').replace('-', '')
                    bar.time = ''
                    bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
                    bar.volume = d.get('turnoverVol', 0)
                except KeyError:
                    print d
                
                flt = {'datetime': bar.datetime}
                self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)            
            
            print u'%s下载完成' %symbol
        else:
            print u'找不到合约%s' %symbol    
Пример #36
0
def loadTdxCsv(fileName, dbName, symbol):
    """将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
    import csv
    
    start = time()
    date_correct = ""
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
    
    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)   
    
    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        bar = VtBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d[1])
        bar.high = float(d[2])
        bar.low = float(d[3])
        bar.close = float(d[4])
        #通达信的夜盘时间按照新的一天计算,此处将其按照当天日期统计,方便后续查阅
        date_temp,time_temp = d[0].strip(' ').replace('\xef\xbb\xbf','').split('-',1)
        if time_temp == '15:00':
            date_correct = date_temp
        if time_temp[:2] == "21" or time_temp[:2] == "22" or time_temp[:2] == "23":
            date_temp = date_correct
            
        bar.date = datetime.strptime(date_temp, '%Y/%m/%d').strftime('%Y%m%d')
        bar.time = time_temp[:2]+':'+time_temp[3:5]+':00'
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
        bar.volume = d[5]

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)  
    
    print u'插入完毕,耗时:%s' % (time()-start)
Пример #37
0
def generateVtBar(row):
    """生成K线"""
    bar = VtBarData()
    
    bar.symbol = row['code']
    bar.exchange = ''
    bar.vtSymbol = bar.symbol
    bar.open = row['open']
    bar.high = row['high']
    bar.low = row['low']
    bar.close = row['close']
    bar.volume = row['volume']
    bar.datetime = datetime.strptime(row['time_key'], '%Y-%m-%d %H:%M:%S')
    bar.date = bar.datetime.strftime("%Y%m%d")
    bar.time = bar.datetime.strftime("%H:%M:%S")
    
    return bar
Пример #38
0
def generateVtBar(row):
    """生成K线"""
    bar = VtBarData()
    
    symbol, exchange = row['symbol'].split('.')
    
    bar.symbol = symbol
    bar.exchange = exchangeMapReverse[exchange]
    bar.vtSymbol = '.'.join([bar.symbol, bar.exchange])
    bar.open = row['open']
    bar.high = row['high']
    bar.low = row['low']
    bar.close = row['close']
    bar.volume = row['volume']
    
    bar.date = str(row['date'])
    bar.time = str(row['time']).rjust(6, '0')
   
    #将bar的时间改成提前一分钟
    hour=bar.time[0:2]
    minute=bar.time[2:4]
    sec=bar.time[4:6]
    if minute=="00":
        minute="59"
        
        h = int(hour)
        if h == 0:
            h = 24
        
        hour=str(h-1).rjust(2,'0')
    else:
        minute=str(int(minute)-1).rjust(2,'0')
    bar.time=hour+minute+sec
   
    bar.datetime = datetime.strptime(' '.join([bar.date, bar.time]), '%Y%m%d %H%M%S')
    
    return bar
Пример #39
0
def loadOKEXCsv(fileName, dbName, symbol):
    """将OKEX导出的csv格式的历史分钟数据插入到Mongo数据库中"""
    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)

    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.reader(open(fileName,"r"))
    for d in reader:
        if len(d[1]) > 10:
            bar = VtBarData()
            bar.vtSymbol = symbol
            bar.symbol = symbol

            bar.datetime = datetime.strptime(d[1], '%Y-%m-%d %H:%M:%S')
            bar.date = bar.datetime.date().strftime('%Y%m%d')
            bar.time = bar.datetime.time().strftime('%H:%M:%S')

            bar.open = float(d[2])
            bar.high = float(d[3])
            bar.low = float(d[4])
            bar.close = float(d[5])

            bar.volume = float(d[6])
            bar.tobtcvolume = float(d[7])

            flt = {'datetime': bar.datetime}
            collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
            print('%s \t %s' % (bar.date, bar.time))

    print u'插入完毕,耗时:%s' % (time()-start)
Пример #40
0
def generateVtBar(row):
    """生成K线"""
    bar = VtBarData()
    
    bar.symbol = row['code']
    bar.exchange = generateExchange(bar.symbol)
    bar.vtSymbol = '.'.join([bar.symbol, bar.exchange])
    bar.open = row['open']
    bar.high = row['high']
    bar.low = row['low']
    bar.close = row['close']
    bar.volume = row['vol']
    bar.datetime = row.name
    bar.date = bar.datetime.strftime("%Y%m%d")
    bar.time = bar.datetime.strftime("%H:%M:%S")
    
    return bar
Пример #41
0
def loadMcCsv(fileName, dbName, symbol):
    """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
    import csv
    
    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
    
    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort']) 
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)   
    
    # 读取数据和插入到数据库
    reader = csv.DictReader(file(fileName, 'r'))
    for d in reader:
        bar = VtBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d['Open'])
        bar.high = float(d['High'])
        bar.low = float(d['Low'])
        bar.close = float(d['Close'])
        bar.date = datetime.strptime(d['Date'], '%Y-%m-%d').strftime('%Y%m%d')
        bar.time = d['Time']
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
        bar.volume = d['TotalVolume']

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)  
        print bar.date, bar.time
    
    print u'插入完毕,耗时:%s' % (time()-start)
Пример #42
0
    def downloadFuturesIntradayBar(self, symbol):
        """下载期货的日内分钟行情"""
        print u'开始下载%s日内分钟行情' %symbol
                
        # 日内分钟行情只有具体合约
        path = 'api/market/getFutureBarRTIntraDay.json'
        
        params = {}
        params['instrumentID'] = symbol
        params['unit'] = 1
        
        data = self.datayesClient.downloadData(path, params)
        
        if data:
            today = datetime.now().strftime('%Y%m%d')
            
            # 创建datetime索引
            self.dbClient[MINUTE_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)], 
                                                                      unique=True)                

            for d in data:
                bar = VtBarData()
                bar.vtSymbol = symbol
                bar.symbol = symbol
                try:
                    bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
                    bar.open = d.get('openPrice', 0)
                    bar.high = d.get('highestPrice', 0)
                    bar.low = d.get('lowestPrice', 0)
                    bar.close = d.get('closePrice', 0)
                    bar.date = today
                    bar.time = d.get('barTime', '')
                    bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M')
                    bar.volume = d.get('totalVolume', 0)
                    bar.openInterest = 0
                except KeyError:
                    print d
                
                flt = {'datetime': bar.datetime}
                self.dbClient[MINUTE_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)            
            
            print u'%s下载完成' %symbol
        else:
            print u'找不到合约%s' %symbol   
Пример #43
0
    def downloadFuturesIntradayBar(self, symbol):
        """下载期货的日内分钟行情"""
        print u'开始下载%s日内分钟行情' % symbol

        # 日内分钟行情只有具体合约
        path = 'api/market/getFutureBarRTIntraDay.json'

        params = {}
        params['instrumentID'] = symbol
        params['unit'] = 1

        data = self.datayesClient.downloadData(path, params)

        if data:
            today = datetime.now().strftime('%Y%m%d')

            # 创建datetime索引
            self.dbClient[MINUTE_DB_NAME][symbol].ensure_index(
                [('datetime', pymongo.ASCENDING)], unique=True)

            for d in data:
                bar = VtBarData()
                bar.vtSymbol = symbol
                bar.symbol = symbol
                try:
                    bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
                        d.get('exchangeCD', ''), '')
                    bar.open = d.get('openPrice', 0)
                    bar.high = d.get('highestPrice', 0)
                    bar.low = d.get('lowestPrice', 0)
                    bar.close = d.get('closePrice', 0)
                    bar.date = today
                    bar.time = d.get('barTime', '')
                    bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
                                                     '%Y%m%d %H:%M')
                    bar.volume = d.get('totalVolume', 0)
                    bar.openInterest = 0
                except KeyError:
                    print d

                flt = {'datetime': bar.datetime}
                self.dbClient[MINUTE_DB_NAME][symbol].update_one(
                    flt, {'$set': bar.__dict__}, upsert=True)

            print u'%s下载完成' % symbol
        else:
            print u'找不到合约%s' % symbol
Пример #44
0
def loadTdxLc1(fileName, dbName, symbol):
    """将通达信导出的lc1格式的历史分钟数据插入到Mongo数据库中"""
    # from struct import *

    start = time()

    print('开始读取通达信Lc1文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))

    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'],
                                 globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    #读取二进制文件
    ofile = open(fileName, 'rb')
    buf = ofile.read()
    ofile.close()

    num = len(buf)
    no = num / 32
    b = 0
    e = 32
    dl = []
    for i in range(no):
        a = unpack('hhfffffii', buf[b:e])
        b = b + 32
        e = e + 32
        bar = VtBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = a[2]
        bar.high = a[3]
        bar.low = a[4]
        bar.close = a[5]
        bar.date = str(int(a[0] / 2048) + 2004) + str(int(
            a[0] % 2048 / 100)).zfill(2) + str(a[0] % 2048 % 100).zfill(2)
        bar.time = str(int(a[1] / 60)).zfill(2) + ':' + str(
            a[1] % 60).zfill(2) + ':00'
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
                                         '%Y%m%d %H:%M:%S')
        bar.volume = a[7]

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)

    print('插入完毕,耗时:%s' % (time() - start))
Пример #45
0
def loadTdxCsv(fileName, dbName, symbol):
    """将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
    import csv

    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)

    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'],
                                 globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        bar = VtBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d[2])
        bar.high = float(d[3])
        bar.low = float(d[4])
        bar.close = float(d[5])
        bar.date = datetime.strptime(d[0], '%Y/%m/%d').strftime('%Y%m%d')
        bar.time = d[1][:2] + ':' + d[1][2:4] + ':00'
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
                                         '%Y%m%d %H:%M:%S')
        bar.volume = d[6]
        bar.openInterest = d[7]

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
        print bar.date, bar.time

    print u'插入完毕,耗时:%s' % (time() - start)
Пример #46
0
def loadMcCsv(fileName, dbName, symbol):
    """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
    import csv

    start = time()
    print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))

    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'],
                                 globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.DictReader(open(fileName, 'r'))
    for d in reader:
        bar = VtBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d['Open'])
        bar.high = float(d['High'])
        bar.low = float(d['Low'])
        bar.close = float(d['Close'])
        bar.date = datetime.strptime(d['Date'], '%Y-%m-%d').strftime('%Y%m%d')
        bar.time = d['Time']
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
                                         '%Y%m%d %H:%M:%S')
        bar.volume = d['TotalVolume']

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
        print(bar.date, bar.time)

    print('插入完毕,耗时:%s' % (time() - start))
Пример #47
0
def generateVtBar(symbol, time, d):
    """生成K线"""
    bar = VtBarData()
    bar.vtSymbol = symbol
    bar.symbol = symbol
    bar.open = float(d['open'])
    bar.high = float(d['high'])
    bar.low = float(d['low'])
    bar.close = float(d['close'])
    bar.date = datetime.strptime(time[0:10], '%Y-%m-%d').strftime('%Y%m%d')
    bar.time = time[11:]
    bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
                                     '%Y%m%d %H:%M:%S')
    bar.volume = d['volume']

    return bar
Пример #48
0
def generateVtBar(row, symbol):
    """生成K线"""
    bar = VtBarData()
    
    bar.symbol = symbol
    bar.vtSymbol = symbol
    bar.open = row['open']
    bar.high = row['high']
    bar.low = row['low']
    bar.close = row['close']
    bar.volume = row['volume']
    bar.datetime = row.name
    bar.date = bar.datetime.strftime("%Y%m%d")
    bar.time = bar.datetime.strftime("%H:%M:%S")
    
    return bar
Пример #49
0
print u'数据下载完成'

# 创建MongoDB连接
client = pymongo.MongoClient('localhost', 27017)
collection = client[DAILY_DB_NAME][vtSymbol]
collection.create_index('datetime')

print u'MongoDB连接成功'

# 将数据插入历史数据库
for row in data.iterrows():
    date = row[0]
    data = row[1]
    
    bar = VtBarData()
    bar.vtSymbol = vtSymbol
    bar.symbol = symbol
    bar.exchange = exchange
    bar.date = date
    bar.datetime = datetime.strptime(date, '%Y-%m-%d')
    bar.open = data['open']
    bar.high = data['high']
    bar.low = data['low']
    bar.close = data['close']
    bar.volume = data['volume']
    
    flt = {'datetime': bar.datetime}
    collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)

print u'数据插入完成'
Пример #50
0
    def onTick(self, tick):
        """收到行情TICK推送(必须由用户继承实现)"""
        # 计算K线
        tickMinute = tick.datetime.minute

        if tickMinute != self.barMinute:    
            if self.bar:
                self.onBar(self.bar)

            bar = VtBarData()              
            bar.vtSymbol = tick.vtSymbol
            bar.symbol = tick.symbol
            bar.exchange = tick.exchange

            bar.open = tick.lastPrice
            bar.high = tick.lastPrice
            bar.low = tick.lastPrice
            bar.close = tick.lastPrice

            bar.date = tick.date
            bar.time = tick.time
            bar.datetime = tick.datetime    # K线的时间设为第一个Tick的时间

            self.bar = bar                  # 这种写法为了减少一层访问,加快速度
            self.barMinute = tickMinute     # 更新当前的分钟
        else:                               # 否则继续累加新的K线
            bar = self.bar                  # 写法同样为了加快速度

            bar.high = max(bar.high, tick.lastPrice)
            bar.low = min(bar.low, tick.lastPrice)
            bar.close = tick.lastPrice
Пример #51
0
def loadTbPlusCsv(fileName, dbName, symbol):
    """将TB极速版导出的csv格式的历史分钟数据插入到Mongo数据库中"""
    import csv

    start = time()
    print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))

    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'],
                                 globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        bar = VtBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d[2])
        bar.high = float(d[3])
        bar.low = float(d[4])
        bar.close = float(d[5])
        bar.date = str(d[0])

        tempstr = str(round(float(d[1]) * 10000)).split(".")[0].zfill(4)
        bar.time = tempstr[:2] + ":" + tempstr[2:4] + ":00"

        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
                                         '%Y%m%d %H:%M:%S')
        bar.volume = d[6]
        bar.openInterest = d[7]
        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
        print(bar.date, bar.time)

    print('插入完毕,耗时:%s' % (time() - start))
Пример #52
0
def generateVtBar(symbol, d):
    """生成K线"""
    bar = VtBarData()
    
    bar.symbol = symbol
    bar.vtSymbol = symbol
    bar.open = d['open']
    bar.high = d['high']
    bar.low = d['low']
    bar.close = d['close']
    bar.volume = d['volume']
    bar.openInterest = d['open_oi']
    bar.datetime = datetime.fromtimestamp(d['datetime']/1000000000)
    bar.date = bar.datetime.strftime("%Y%m%d")
    bar.time = bar.datetime.strftime("%H:%M:%S")
    
    return bar
Пример #53
0
def downloadEquityDailyBarts(self, symbol):
    """
    下载股票的日行情,symbol是股票代码
    """
    print('开始下载%s日行情' % symbol)

    # 查询数据库中已有数据的最后日期
    cl = self.dbClient[DAILY_DB_NAME][symbol]
    cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
    if cx.count():
        last = cx[0]
    else:
        last = ''
    # 开始下载数据
    import tushare as ts

    if last:
        start = last['date'][:4] + '-' + last['date'][4:6] + '-' + last[
            'date'][6:]

    data = ts.get_k_data(symbol, start)

    if not data.empty:
        # 创建datetime索引
        self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
            [('datetime', pymongo.ASCENDING)], unique=True)

        for index, d in data.iterrows():
            bar = VtBarData()
            bar.vtSymbol = symbol
            bar.symbol = symbol
            try:
                bar.open = d.get('open')
                bar.high = d.get('high')
                bar.low = d.get('low')
                bar.close = d.get('close')
                bar.date = d.get('date').replace('-', '')
                bar.time = ''
                bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
                bar.volume = d.get('volume')
            except KeyError:
                print(d)

            flt = {'datetime': bar.datetime}
            self.dbClient[DAILY_DB_NAME][symbol].update_one(
                flt, {'$set': bar.__dict__}, upsert=True)

        print('%s下载完成' % symbol)
    else:
        print('找不到合约%s' % symbol)
Пример #54
0
 def loadRqBar(self, symbol, days):
     """从RQData加载K线数据"""
     endDate = datetime.now()
     startDate = endDate - timedelta(days)
     
     df = self.rq.get_price(symbol.upper(), 
                            frequency='1m', 
                            fields=['open', 'high', 'low', 'close', 'volume'],
                            start_date=startDate,
                            end_date=endDate)
     
     l = []
     
     for ix, row in df.iterrows():
         bar = VtBarData()
         bar.symbol = symbol
         bar.vtSymbol = symbol
         bar.open = row['open']
         bar.high = row['high']
         bar.low = row['low']
         bar.close = row['close']
         bar.volume = row['volume']
         bar.datetime = row.name
         bar.date = bar.datetime.strftime("%Y%m%d")
         bar.time = bar.datetime.strftime("%H:%M:%S")
         
         l.append(bar)
     
     return l
Пример #55
0
def loadOKCsv(fileName, dbName, symbol):
    """将OKEX导出的csv格式的历史分钟数据插入到Mongo数据库中"""
    start = time()
    print('开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))

    # 锁定集合,并创建索引
    client = pymongo.MongoClient(globalSetting['mongoHost'],
                                 globalSetting['mongoPort'])
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.reader(open(fileName, "r"))

    for d in reader:
        print(d)
        if len(d[0]) > 10:
            bar = VtBarData()
            bar.vtSymbol = symbol
            bar.symbol, bar.exchange = symbol.split(':')

            bar.datetime = datetime.strptime(d[0], '%Y-%m-%d %H:%M:%S')
            bar.date = bar.datetime.date().strftime('%Y%m%d')
            bar.time = bar.datetime.time().strftime('%H:%M:%S')
            print(d[1])
            if d[1]:
                bar.high = float(d[1])
                bar.low = float(d[2])
                bar.open = float(d[3])
                bar.close = float(d[4])

                bar.amount = float(d[5])
                bar.volume = float(d[6])

            flt = {'datetime': bar.datetime}
            collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
            print('%s \t %s' % (bar.date, bar.time))

    print('插入完毕,耗时:%s' % (time() - start))
Пример #56
0
def generateVtBar(symbol, d):
    """生成K线"""
    bar = VtBarData()
    
    bar.symbol = symbol
    bar.vtSymbol = symbol
    bar.datetime = datetime.datetime.strptime(d['time_open'], '%Y-%m-%dT%H:%M:%S.%f0Z')
    bar.date = bar.datetime.strftime('%Y%m%d')
    bar.time = bar.datetime.strftime('%H:%M:%S')
    bar.open = d['price_open']
    bar.high = d['price_high']
    bar.low = d['price_low']
    bar.close = d['price_close']
    bar.volume = d['volume_traded']
    
    return bar
Пример #57
0
    def generateBar(self, row, symbol):
        """生成K线对象"""
        bar = VtBarData()

        bar.symbol = symbol
        bar.vtSymbol = symbol
        bar.open = row['open']
        bar.high = row['high']
        bar.low = row['low']
        bar.close = row['close']
        bar.volume = row['volume']
        bar.datetime = row.name
        bar.date = bar.datetime.strftime("%Y%m%d")
        bar.time = bar.datetime.strftime("%H:%M:%S")

        return bar
Пример #58
0
 def onBar(self, bar):
     """收到Bar推送(必须由用户继承实现)"""
     # 如果当前是一个5分钟走完(分钟线的时间戳是当前分钟的开始时间戳,因此要+1)
     if (bar.datetime.minute + 1) % 5 == 0:
         # 如果已经有聚合5分钟K线
         if self.fiveBar:
             # 将最新分钟的数据更新到目前5分钟线中
             fiveBar = self.fiveBar
             fiveBar.high = max(fiveBar.high, bar.high)
             fiveBar.low = min(fiveBar.low, bar.low)
             fiveBar.close = bar.close
             
             # 推送5分钟线数据
             self.onFiveBar(fiveBar)
             
             # 清空5分钟线数据缓存
             self.fiveBar = None
     else:
         # 如果没有缓存则新建
         if not self.fiveBar:
             fiveBar = VtBarData()
             
             fiveBar.vtSymbol = bar.vtSymbol
             fiveBar.symbol = bar.symbol
             fiveBar.exchange = bar.exchange
         
             fiveBar.open = bar.open
             fiveBar.high = bar.high
             fiveBar.low = bar.low
             fiveBar.close = bar.close
         
             fiveBar.date = bar.date
             fiveBar.time = bar.time
             fiveBar.datetime = bar.datetime 
             
             self.fiveBar = fiveBar
         else:
             fiveBar = self.fiveBar
             fiveBar.high = max(fiveBar.high, bar.high)
             fiveBar.low = min(fiveBar.low, bar.low)
             fiveBar.close = bar.close