Beispiel #1
0
def loadTdxCsv(fileName, dbName, symbol):
    """将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
    import csv
    
    start = time()
    print(u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
    
    # 锁定集合,并创建索引
    host, port, logging = loadMongoSetting()
    
    client = pymongo.MongoClient(host, port)    
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)   
    
    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        bar = CtaBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d[2])
        bar.high = float(d[3])
        bar.low = float(d[4])
        bar.close = float(d[5])
        bar.date = datetime.strptime(d[0], '%Y/%m/%d').strftime('%Y%m%d')
        bar.time = d[1][:2]+':'+d[1][2:4]+':00'
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
        bar.volume = d[6]
        bar.openInterest = d[7]

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)  
        print(bar.date, bar.time)
    
    print(u'插入完毕,耗时:%s' % (time()-start))
Beispiel #2
0
def loadMcCsv(fileName, dbName, symbol):
    """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
    import csv
    
    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
    
    # 锁定集合,并创建索引
    host, port = loadMongoSetting()
    
    client = pymongo.MongoClient(host, port)    
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)   
    
    # 读取数据和插入到数据库
    reader = csv.DictReader(file(fileName, 'r'))
    for d in reader:
        bar = CtaBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d['Open'])
        bar.high = float(d['High'])
        bar.low = float(d['Low'])
        bar.close = float(d['Close'])
        bar.date = datetime.strptime(d['Date'], '%Y/%m/%d').strftime('%Y%m%d')
        bar.time = d['Time']
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
        bar.volume = d['TotalVolume']

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)  
        print bar.date, bar.time
    
    print u'插入完毕,耗时:%s' % (time()-start)
Beispiel #3
0
    def __init__(self):
        """Constructor"""
        host, port, logging = loadMongoSetting()

        print(host)
        self.dbClient = pymongo.MongoClient(host, port)
        self.datayesClient = DatayesClient()
Beispiel #4
0
def loadMcCsv(fileName, dbName, symbol):
    """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
    import csv

    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)

    # 锁定集合,并创建索引
    host, port = loadMongoSetting()

    client = pymongo.MongoClient(host, port)
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.DictReader(file(fileName, 'r'))
    for d in reader:
        bar = CtaBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d['Open'])
        bar.high = float(d['High'])
        bar.low = float(d['Low'])
        bar.close = float(d['Close'])
        bar.date = datetime.strptime(d['Date'], '%Y/%m/%d').strftime('%Y%m%d')
        bar.time = d['Time']
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
                                         '%Y%m%d %H:%M:%S')
        bar.volume = d['TotalVolume']

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
        print bar.date, bar.time

    print u'插入完毕,耗时:%s' % (time() - start)
Beispiel #5
0
    def loadHistoryData(self):
        """载入历史数据"""
        """load historical data"""

        host, port = loadMongoSetting()

        self.dbClient = pymongo.MongoClient(host, port)
        collection = self.dbClient[self.dbName][self.symbol]

        self.output("Start loading historical data")

        # 首先根据回测模式,确认要使用的数据类
        # Choose data type based on backtest mode
        if self.mode == self.BAR_MODE:
            dataClass = CtaBarData
        else:
            dataClass = CtaTickData

        # 载入初始化需要用的数据
        # Load initialised data

        # $gte means "greater and equal to"
        # $lt means "less than"
        flt = {
            'datetime': {
                '$gte': self.dataStartDate,
                '$lt': self.strategyStartDate
            }
        }
        initCursor = collection.find(flt)

        # 将数据从查询指针中读取出,并生成列表
        # Read data from cursor, generate a list
        self.initData = []  # Empty "initData" list
        for d in initCursor:
            data = dataClass()
            data.__dict__ = d
            self.initData.append(data)

        # 载入回测数据
        # Load backtest data (exclude initialised data)
        if not self.dataEndDate:

            # If "End Date" is not set, retreat data up to today
            flt = {'datetime': {'$gte': self.strategyStartDate}}
        else:
            flt = {
                'datetime': {
                    '$gte': self.strategyStartDate,
                    '$lte': self.dataEndDate
                }
            }
        self.dbCursor = collection.find(flt)

        self.output("Data loading complete, data volumn: %s" %
                    (initCursor.count() + self.dbCursor.count()))
Beispiel #6
0
    def loadHistoryData(self):
        """载入历史数据"""
        """load historical data"""

        host, port, logging = loadMongoSetting()

        self.dbClient = pymongo.MongoClient(host, port)
        collection = self.dbClient[self.dbName][self.symbol]

        # Load historical data of information symbols, construct a dictionary of Database
        # Values of dictionary are mongo.Client.
        info_collection = {}
        if self.MultiOn is True:
            for DBname, symbol in self.info_symbols:
                info_collection[DBname + " " +
                                symbol] = self.dbClient[DBname][symbol]

        self.output("Start loading historical data")

        # 首先根据回测模式,确认要使用的数据类
        # Choose data type based on backtest mode
        if self.mode == self.BAR_MODE:
            self.dataClass = CtaBarData
            self.func = self.newBar
        else:
            self.dataClass = CtaTickData
            self.func = self.newTick

        # Load initializing data
        self.loadInitData(collection, inf=info_collection)

        # 载入回测数据
        # Load backtest data (exclude initializing data)
        if not self.dataEndDate:
            # If "End Date" is not set, retreat data up to today
            flt = {'datetime': {'$gte': self.strategyStartDate}}
        else:
            flt = {
                'datetime': {
                    '$gte': self.strategyStartDate,
                    '$lte': self.dataEndDate
                }
            }
        self.dbCursor = collection.find(flt)

        if self.MultiOn is True:
            for db in info_collection:
                self.InfoCursor[db] = info_collection[db].find(flt)
            self.output(
                "Data loading completed, data volumn: %s" % (self.initCursor.count() + self.dbCursor.count() + \
                                                             sum([i.count() for i in self.InfoCursor.values()])))
        else:
            self.output("Data loading completed, data volumn: %s" %
                        (self.initCursor.count() + self.dbCursor.count()))
Beispiel #7
0
def loadTBCsv(fileName, dbName, symbol):
    """将TradeBlazer导出的csv格式的历史数据插入到Mongo数据库中
        数据样本:
        //时间,开盘价,最高价,最低价,收盘价,成交量,持仓量
        2017/04/05 09:00,3200,3240,3173,3187,312690,2453850

        20150105,0.0911,2572,2574,2571,2573,27744,2570474
    """
    import csv

    start = time()
    print(u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))

    # 锁定集合,并创建索引
    host, port, logging = loadMongoSetting()

    client = pymongo.MongoClient(host, port)
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        if len(d[0]) == 8:
            bar = CtaBarData()
            bar.vtSymbol = symbol
            bar.symbol = symbol

            # bar.datetime = datetime.strptime(d[0], '%Y/%m/%d %H:%M')
            # bar.date = bar.datetime.date().strftime('%Y%m%d')
            # bar.time = bar.datetime.time().strftime('%H:%M:%S')

            deltaMinutes = float(d[1]) * 10000
            bar.datetime = datetime.strptime(d[0], '%Y%m%d') + timedelta(
                hours=int(float(d[1]) * 100),
                minutes=int(float(d[1]) * 10000) % 100)
            bar.date = bar.datetime.date().strftime('%Y%m%d')
            bar.time = bar.datetime.time().strftime('%H:%M:%S')

            bar.open = float(d[1 + 1])
            bar.high = float(d[2 + 1])
            bar.low = float(d[3 + 1])
            bar.close = float(d[4 + 1])

            bar.volume = float(d[5 + 1])
            bar.openInterest = float(d[6 + 1])

            flt = {'datetime': bar.datetime}
            collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
            print('%s \t %s' % (bar.date, bar.time))

    print(u'插入完毕,耗时:%s' % (time() - start))
Beispiel #8
0
def loadCsv(filename, dbName, symbol, exch):
    start = time()
    #print u'Start'

    host, port = loadMongoSetting()
    #print host, port

    client = pymongo.MongoClient(host, port)
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
    reader = csv.reader(file(filename, 'r'))
    first_line = True
    for d in reader:
        try:
            if first_line or d[6] == '0' or d[7] == '0' or d[12] == '0' or d[
                    13] == '0':
                first_line = False
                continue
        except IndexError:
            break
        tick = CtaTickData()
        tick.vtSymbol = symbol
        tick.symbol = symbol
        tick.exchange = exch
        tick.lastPrice = float(d[2])
        tick.volume = float(d[3])
        tick.openInterest = float(d[5])
        tick.date = datetime.strptime(d[0], '%Y-%m-%d').strftime('%Y%m%d')
        tick.time = d[1]
        tick.datetime = datetime.strptime(tick.date + ' ' + tick.time,
                                          '%Y%m%d %H:%M:%S')

        tick.bidPrice1 = float(d[12])
        #tick.bidPrice2=float(d[14])
        #tick.bidPrice3=float(d[16])

        tick.askPrice1 = float(d[6])
        #tick.askPrice2=float(d[8])
        #tick.askPrice3=float(d[10])

        tick.bidVolume1 = float(d[13])
        #tick.bidVolume2=float(d[15])
        #tick.bidVolume3=float(d[17])

        tick.askVolume1 = float(d[7])
        #tick.askVolume2=float(d[9])
        #tick.askVolume3=float(d[11])

        flt = {'datetime': tick.datetime}
        collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
        print symbol, tick.date, tick.time
    print u'OK', filename
Beispiel #9
0
def loadCsv(filename, dbName, symbol):

    #print u'Start'

    host, port, logging = loadMongoSetting()
    #print host, port

    client = pymongo.MongoClient(host, port)
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
    reader = csv.reader(file(filename, 'r'))
    first_line = True
    lastdatetime = None
    temp_list = []
    for d in reader:
        try:
            if first_line or d[6] == '0' or d[7] == '0' or d[12] == '0' or d[
                    13] == '0':
                first_line = False
                continue
        except IndexError:
            break
        temp = {'lastPrice': 0.0, 'datetime': None}
        date = datetime.strptime(d[0], '%Y-%m-%d').strftime('%Y%m%d')
        time = datetime.strptime(d[1], '%H:%M:%S').strftime('%H:%M:%S')
        temp['datetime'] = datetime.strptime(date + ' ' + time,
                                             '%Y%m%d %H:%M:%S')
        if lastdatetime != temp['datetime']:
            temp['lastPrice'] = float(d[2])
            temp_list.append(temp)
            lastdatetime = temp['datetime']
        else:
            temp1 = temp_list.pop()
            t = temp1['lastPrice']
            temp1['lastPrice'] = (float(t) + float(d[2])) / 2
            temp_list.append(temp1)
            lastdatetime = temp1['datetime']
        #print temp['datetime']
    try:
        collection.insert(temp_list)
        print u'OK', filename
    except:
        for i in temp_list:
            try:
                collection.insert([i])
                print u'OK', i['datetime']
            except:
                print u'Fail', i['datetime']
                continue
    def loadHistoryData(self):
        """载入历史数据"""
        host, port = loadMongoSetting()

        self.dbClient = pymongo.MongoClient(host, port)
        self.output(u'开始载入数据')

        for symbol in self.symbols:

            self.output(u"载入历史数据" + str(symbol))

            collection = self.dbClient[self.dbName][symbol]

            # 载入回测数据
            if not self.dataEndDate:
                flt = {'datetime': {'$gte': self.strategyStartDate}}  # 数据过滤条件
            else:
                flt = {
                    'datetime': {
                        '$gte': self.strategyStartDate,
                        '$lte': self.dataEndDate
                    }
                }
            dbCursor = collection.find(flt)
            initData = []
            for d in dbCursor:
                initData.append(d)

            if self.dataframe.empty:
                self.dataframe = pd.DataFrame(initData)
            else:
                self.dataframe = pd.concat(
                    [self.dataframe, pd.DataFrame(initData)])

            #这里只管喂就好了,初始化的逻辑交给策略
        df = self.dataframe.drop_duplicates()
        del df['_id']
        df = to_transed_eve(df)
        df = df.sort_values('datetime')
        index = pd.Index(np.arange(df.count()[0]))
        df.index = index
        df[['open', 'close', 'low',
            'high']] = df[['open', 'close', 'low', 'high']].applymap(float)
        df[['volume']] = df[['volume']].applymap(int)

        self.dataframe = df

        self.output(u'载入完成,数据量:%s' % (self.dataframe.count()[0]))
Beispiel #11
0
    def loadHistoryData(self):
        """载入历史数据"""
        """load historical data"""

        host, port, logging = loadMongoSetting()

        self.dbClient = pymongo.MongoClient(host, port)
        collection = self.dbClient[self.dbName][self.symbol]

        # Load historical data of information symbols, construct a dictionary of Database
        # Values of dictionary are mongo.Client.
        info_collection = {}
        if self.MultiOn is True:
            for DBname, symbol in self.info_symbols:
                info_collection[DBname + " " + symbol] = self.dbClient[DBname][symbol]

        self.output("Start loading historical data")

        # 首先根据回测模式,确认要使用的数据类
        # Choose data type based on backtest mode
        if self.mode == self.BAR_MODE:
            self.dataClass = CtaBarData
            self.func      = self.newBar
        else:
            self.dataClass = CtaTickData
            self.func = self.newTick

        # Load initializing data
        self.loadInitData(collection, inf=info_collection)

        # 载入回测数据
        # Load backtest data (exclude initializing data)
        if not self.dataEndDate:
            # If "End Date" is not set, retreat data up to today
            flt = {'datetime': {'$gte': self.strategyStartDate}}
        else:
            flt = {'datetime': {'$gte': self.strategyStartDate,
                                '$lte': self.dataEndDate}}
        self.dbCursor = collection.find(flt)

        if self.MultiOn is True:
            for db in info_collection:
                self.InfoCursor[db] = info_collection[db].find(flt)
            self.output(
                "Data loading completed, data volumn: %s" % (self.initCursor.count() + self.dbCursor.count() + \
                                                             sum([i.count() for i in self.InfoCursor.values()])))
        else:
            self.output("Data loading completed, data volumn: %s" % (self.initCursor.count() + self.dbCursor.count()))
Beispiel #12
0
    def dbConnect(self):
        """连接MongoDB数据库"""
        if not self.dbClient:
            # 读取MongoDB的设置
            host, port = loadMongoSetting()
                
            try:
                # 设置MongoDB操作的超时时间为0.5秒
                self.dbClient = MongoClient(host, port, serverSelectionTimeoutMS=500)
                
                # 调用server_info查询服务器状态,防止服务器异常并未连接成功
                self.dbClient.server_info()

                self.writeLog(u'MongoDB连接成功')
            except ConnectionFailure:
                self.writeLog(u'MongoDB连接失败')
Beispiel #13
0
def loadMinuteTxt(fileName, dbName, symbol):
    """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
    import csv
    import pandas as pd

    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)

    # 锁定集合,并创建索引
    host, port, logging = loadMongoSetting()

    client = pymongo.MongoClient(host, port)
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    lines = open(fileName, 'r').readlines()
    lines = [line.replace('\t', ',') for line in lines]
    newfile = open('swap.txt', 'w')
    newfile.writelines('Date,Time,Open,High,Low,Close,Vol,Val\r\n')
    newfile.writelines(lines)
    newfile.close()
    reader = csv.DictReader(file(
        'swap.txt',
        'r',
    ))

    for d in reader:
        bar = CtaBarData()
        bar.vtSymbol = symbol
        bar.symbol = symbol
        bar.open = float(d['Open'])
        bar.high = float(d['High'])
        bar.low = float(d['Low'])
        bar.close = float(d['Close'])
        bar.date = datetime.strptime(d['Date'], '%Y/%m/%d').strftime('%Y%m%d')
        bar.time = d['Time']
        bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
                                         '%Y%m%d %H:%M')
        bar.volume = d['Vol']
        bar.openInterest = d['Val']

        flt = {'datetime': bar.datetime}
        collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
        print bar.date, bar.time

    print u'插入完毕,耗时:%s' % (time() - start)
Beispiel #14
0
    def loadHistoryData(self):
        """载入历史数据"""
        host, port, logging = loadMongoSetting()

        self.dbClient = pymongo.MongoClient(host, port)
        collection = self.dbClient[self.dbName][self.symbol]

        self.output(u'开始载入数据')

        # 首先根据回测模式,确认要使用的数据类
        if self.mode == self.BAR_MODE:
            dataClass = CtaBarData
            func = self.newBar
        else:
            dataClass = CtaTickData
            func = self.newTick

        # 载入初始化需要用的数据
        flt = {
            'datetime': {
                '$gte': self.dataStartDate,
                '$lt': self.strategyStartDate
            }
        }
        initCursor = collection.find(flt)

        # 将数据从查询指针中读取出,并生成列表
        self.initData = []  # 清空initData列表
        for d in initCursor:
            data = dataClass()
            data.__dict__ = d
            self.initData.append(data)

        # 载入回测数据
        if not self.dataEndDate:
            flt = {'datetime': {'$gte': self.strategyStartDate}}  # 数据过滤条件
        else:
            flt = {
                'datetime': {
                    '$gte': self.strategyStartDate,
                    '$lte': self.dataEndDate
                }
            }
        self.dbCursor = collection.find(flt)

        self.output(u'载入完成,数据量:%s' %
                    (initCursor.count() + self.dbCursor.count()))
def loadTBCsv(fileName, dbName, symbol):
    """将TradeBlazer导出的csv格式的历史数据插入到Mongo数据库中
        数据样本:
        //时间,开盘价,最高价,最低价,收盘价,成交量,持仓量
        2017/04/05 09:00,3200,3240,3173,3187,312690,2453850
    """
    import csv
    '''2017052500 Add by hetajen begin'''
    start = time.time()
    '''2017052500 Add by hetajen end'''
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)

    # 锁定集合,并创建索引
    host, port, logging = loadMongoSetting()

    client = pymongo.MongoClient(host, port)
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        if len(d[0]) > 10:
            bar = CtaBarData()
            bar.vtSymbol = symbol
            bar.symbol = symbol
            '''2017052500 Add by hetajen begin'''
            bar.datetime = datetime.datetime.strptime(d[0], '%Y/%m/%d %H:%M')
            '''2017052500 Add by hetajen end'''
            bar.date = bar.datetime.date().strftime('%Y%m%d')
            bar.time = bar.datetime.time().strftime('%H:%M:%S')

            bar.open = float(d[1])
            bar.high = float(d[2])
            bar.low = float(d[3])
            bar.close = float(d[4])

            bar.volume = float(d[5])
            bar.openInterest = float(d[6])

            flt = {'datetime': bar.datetime}
            collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
            print '%s \t %s' % (bar.date, bar.time)
    '''2017052500 Add by hetajen begin'''
    print u'插入完毕,耗时:%s' % (time.time() - start)
    '''2017052500 Add by hetajen end'''
Beispiel #16
0
    def dbConnect(self):
        """连接MongoDB数据库"""

        # 读取数据库配置的方法,已转移至vtFunction
        # # 载入json文件
        # fileName = 'Mongo_connect.json'
        # fileName = os.getcwd() + '\\mongoConnect\\' + fileName
        #
        # try:
        #     f = file(fileName)
        # except IOError:
        #     self.writeLog(u'读取MongoDB连接配置出错,请检查')
        #     return
        #
        # # 解析json文件
        # setting = json.load(f)
        # try:
        #     IP = str(setting['IP'])
        #     replicaset = str(setting['replicaset'])
        #     readPreference = str(setting['readPreference'])
        #     database = str(setting['db'])
        #     userID = str(setting['userID'])
        #     password = str(setting['password'])
        #
        # except KeyError:
        #     self.writeLog(u'MongoDB连接配置缺少字段,请检查')

        if not self.dbClient:
            # 读取MongoDB的设置
            host, port, replicaset, readPreference, database, userID, password = loadMongoSetting()
            try:
                # 设置MongoDB操作的超时时间为0.5秒
                # self.dbClient = pymongo.MongoClient(host+':'+str(port), replicaset=replicaset,readPreference=readPreference, serverSelectionTimeoutMS=500)
                # db = self.dbClient[database]
                # db.authenticate(userID, password)

                self.dbClient = pymongo.MongoClient(host, port, serverSelectionTimeoutMS=500)

                # 调用server_info查询服务器状态,防止服务器异常并未连接成功
                self.dbClient.server_info()

                self.writeLog(u"MongoDB连接成功")
            except ConnectionFailure:
                self.writeLog(u"MongoDB连接失败")
            except ValueError:
                self.writeLog(u"MongoDB连接配置字段错误,请检查")
Beispiel #17
0
    def dbConnect(self):
        """连接MongoDB数据库"""
        if not self.dbClient:
            # 读取MongoDB的设置
            host, port = loadMongoSetting()

            try:
                # 设置MongoDB操作的超时时间为0.5秒
                self.dbClient = MongoClient(host,
                                            port,
                                            serverSelectionTimeoutMS=500)

                # 调用server_info查询服务器状态,防止服务器异常并未连接成功
                self.dbClient.server_info()

                self.writeLog(u'MongoDB连接成功')
            except ConnectionFailure:
                self.writeLog(u'MongoDB连接失败')
Beispiel #18
0
def loadTBCsv(fileName, dbName, symbol):
    """将TradeBlazer导出的csv格式的历史数据插入到Mongo数据库中
        数据样本:
        //时间,开盘价,最高价,最低价,收盘价,成交量,持仓量
        2017/04/05 09:00,3200,3240,3173,3187,312690,2453850
    """
    import csv
    
    start = time()
    print(u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
    
    # 锁定集合,并创建索引
    host, port, logging = loadMongoSetting()
    
    client = pymongo.MongoClient(host, port)    
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
    
    # 读取数据和插入到数据库
    reader = csv.reader(file(fileName, 'r'))
    for d in reader:
        if len(d[0]) > 10:
            bar = CtaBarData()
            bar.vtSymbol = symbol
            bar.symbol = symbol
            
            bar.datetime = datetime.strptime(d[0], '%Y/%m/%d %H:%M')
            bar.date = bar.datetime.date().strftime('%Y%m%d')
            bar.time = bar.datetime.time().strftime('%H:%M:%S')
            
            bar.open = float(d[1])
            bar.high = float(d[2])
            bar.low = float(d[3])
            bar.close = float(d[4])
            
            bar.volume = float(d[5])
            bar.openInterest = float(d[6])

            flt = {'datetime': bar.datetime}
            collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
            print('%s \t %s' % (bar.date, bar.time))
    
    print(u'插入完毕,耗时:%s' % (time()-start))
def loadMcCsv(fileName, dbName, symbol):
    import csv
    from time import time
    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)

    # 锁定集合,并创建索引
    host, port = loadMongoSetting()

    client = pymongo.MongoClient(host, port)
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    # 读取数据和插入到数据库
    reader = csv.DictReader(file(fileName, 'r'))
    for d in reader:
        tick = CtaTickData()
        tick.vtSymbol = symbol
        tick.symbol = d['CONTRACTID']
        tick.lastPrice = float(d['LASTPX'])
        tick.bidPrice1 = float(d['B1'])
        tick.askPrice1 = float(d['S1'])
        tick.bidVolume1 = float(d['BV1'])
        tick.askVolume1 = float(d['SV1'])
        tick.volume = float(d['LASTQTY'])
        tick.openInterest = float(d['OPENINTS'])
        tick.date = float(d['TRADINGDATE'])
        tick.datetime = datetime.strptime(d['TDATETIME'][:19],
                                          '%Y-%m-%d %H:%M:%S')

        tick.date = datetime.strptime(d['TRADINGDATE'],
                                      '%Y%m%d').strftime('%Y%m%d')
        tick.upperLimit = d['RISELIMIT']
        tick.lowerLimit = d['FALLLIMIT']
        tick.time = d['TDATETIME']

        tick.exchange = d['MARKET']

        flt = {'datetime': tick.datetime}
        collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
        print u'正在插入:', tick.date, tick.time, u'的历史数据到mongodb'

    print u'插入完毕,耗时:%s' % (time() - start)
    def dbConnect(self):
        """连接MongoDB数据库"""

        # 载入json文件
        if not self.dbClient:
            # 读取MongoDB的设置
            host, port, replicaset, readPreference, database, userID, password = loadMongoSetting()
            try:
                self.dbClient = pymongo.MongoClient(host+':'+str(port), replicaset=replicaset,readPreference=readPreference)
                db = self.dbClient[database]
                db.authenticate(userID, password)
                # self.dbClient = MongoClient(host, port)
                self.writeCtaLog(u'MongoDB连接成功')
            except ConnectionFailure:
                self.writeCtaLog(u'MongoDB连接失败')
            except ValueError:
                self.writeCtaLog(u'MongoDB连接配置字段错误,请检查')

            self.mainEngine.dbClient = self.dbClient        # 主引擎数据库连接
    def dbConnect(self):
        """连接MongoDB数据库"""

        # 读取数据库配置的方法,已转移至vtFunction

        if not self.dbClient:
            # 读取MongoDB的设置
            settingFileName = "VT_setting.json"
            settingFileName = os.path.dirname(os.getcwd()) + "/" + settingFileName
            host, port, replicaset, readPreference, database, userID, password = loadMongoSetting(settingFileName)
            try:
                # self.dbClient = pymongo.MongoClient(host+':'+str(port), replicaset=replicaset,readPreference=readPreference)
                # db = self.dbClient[database]
                # db.authenticate(userID, password)
                self.dbClient = pymongo.MongoClient(host, port, serverSelectionTimeoutMS=500)
                print u'MongoDB连接成功'
            except ConnectionFailure:
                print u'MongoDB连接失败'
            except ValueError:
                print u'MongoDB连接配置字段错误,请检查'
Beispiel #22
0
    def dbConnect(self):
        """连接MongoDB数据库"""
        if not self.dbClient:
            # 读取MongoDB的设置
            host, port, logging = loadMongoSetting()

            try:
                # 设置MongoDB操作的超时时间为0.5秒
                self.dbClient = MongoClient(host, port, connectTimeoutMS=500)

                # 调用server_info查询服务器状态,防止服务器异常并未连接成功
                self.dbClient.server_info()

                self.writeLog(text.DATABASE_CONNECTING_COMPLETED)

                # 如果启动日志记录,则注册日志事件监听函数
                if logging:
                    self.eventEngine.register(EVENT_LOG, self.dbLogging)

            except ConnectionFailure:
                self.writeLog(text.DATABASE_CONNECTING_FAILED)
Beispiel #23
0
    def dbConnect(self):
        """连接MongoDB数据库"""
        if not self.dbClient:
            # 读取MongoDB的设置
            host, port, logging = loadMongoSetting()
                
            try:
                # 设置MongoDB操作的超时时间为0.5秒
                self.dbClient = MongoClient(host, port, connectTimeoutMS=500)
                
                # 调用server_info查询服务器状态,防止服务器异常并未连接成功
                self.dbClient.server_info()

                self.writeLog(text.DATABASE_CONNECTING_COMPLETED)
                
                # 如果启动日志记录,则注册日志事件监听函数
                if logging:
                    self.eventEngine.register(EVENT_LOG, self.dbLogging)
                    
            except ConnectionFailure:
                self.writeLog(text.DATABASE_CONNECTING_FAILED)
Beispiel #24
0
    def loadHistoryData(self):
        """载入历史数据"""
        host, port, logging = loadMongoSetting()
        
        self.dbClient = pymongo.MongoClient(host, port)
        collection = self.dbClient[self.dbName][self.symbol]

        self.output(u'开始载入数据')
      
        # 首先根据回测模式,确认要使用的数据类
        if self.mode == self.BAR_MODE:
            dataClass = CtaBarData
            func = self.newBar
        else:
            dataClass = CtaTickData
            func = self.newTick

        # 载入初始化需要用的数据
        flt = {'datetime':{'$gte':self.dataStartDate,
                           '$lt':self.strategyStartDate}}        
        initCursor = collection.find(flt)
        
        # 将数据从查询指针中读取出,并生成列表
        self.initData = []              # 清空initData列表
        for d in initCursor:
            data = dataClass()
            data.__dict__ = d
            self.initData.append(data)      
        
        # 载入回测数据
        if not self.dataEndDate:
            flt = {'datetime':{'$gte':self.strategyStartDate}}   # 数据过滤条件
        else:
            flt = {'datetime':{'$gte':self.strategyStartDate,
                               '$lte':self.dataEndDate}}  
        self.dbCursor = collection.find(flt)
        
        self.output(u'载入完成,数据量:%s' %(initCursor.count() + self.dbCursor.count()))
Beispiel #25
0
def loadMcCsvTick(fileName, dbName, symbol):
    """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
    import csv

    start = time()
    print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
    """
    csv_file = open(fileName, 'r')
    reader = csv.DictReader(csv_file)

    for d in reader:
        print float(d['\xc9\xea\xc2\xf2\xbc\xdb\xd2\xbb'])
    csv_file.close()
    """

    # 锁定集合,并创建索引
    host, port = loadMongoSetting()

    client = pymongo.MongoClient(host, port)
    collection = client[dbName][symbol]
    collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)

    csv_file = open(fileName, 'r')
    reader = csv.DictReader(csv_file)
    for d in reader:
        tick = CtaTickData()
        tick.vtSymbol = symbol
        tick.symbol = symbol
        tick.askPrice1 = float(d['\xc9\xea\xc2\xf2\xbc\xdb\xd2\xbb'])  #申买价一
        tick.askVolume1 = int(d['\xc9\xea\xc2\xf2\xc1\xbf\xd2\xbb'])  #申买量一
        tick.bidPrice1 = float(d['\xc9\xea\xc2\xf4\xbc\xdb\xd2\xbb'])  #申卖价一
        tick.bidVolume1 = int(d['\xc9\xea\xc2\xf4\xc1\xbf\xd2\xbb'])  #申卖量一
        tick.lastPrice = float(d['\xd7\xee\xd0\xc2\xbc\xdb'])  #最新价
        tick.upperLimit = float(d['\xd5\xc7\xcd\xa3\xb0\xe5\xbc\xdb'])  #涨停板价
        tick.lowerLimit = float(d['\xb5\xf8\xcd\xa3\xb0\xe5\xbc\xdb'])  #跌停板价
        tick.volume = int(float(d['\xb3\xd6\xb2\xd6\xc1\xbf']))  #持仓量
        tick.high = float(d['\xd7\xee\xb8\xdf\xbc\xdb'])  # 最高价
        tick.low = float(d['\xd7\xee\xb5\xcd\xbc\xdb'])  # 最低价

        tick.date = d['\xbd\xbb\xd2\xd7\xc8\xd5']  #交易日
        #tick.date = datetime.strptime(d['date'], '%Y%m%d').strftime('%Y%m%d')
        if d['\xd7\xee\xba\xf3\xd0\xde\xb8\xc4\xba\xc1\xc3\xeb'] == '500':  #最后修改毫秒
            tick.time = d[
                '\xd7\xee\xba\xf3\xd0\xde\xb8\xc4\xca\xb1\xbc\xe4'] + '.5'  #最后修改时间
        else:
            tick.time = d[
                '\xd7\xee\xba\xf3\xd0\xde\xb8\xc4\xca\xb1\xbc\xe4'] + '.0'  #最后修改时间
        tick.datetime = datetime.strptime(tick.date + ' ' + tick.time,
                                          '%Y%m%d %H:%M:%S.%f')

        #21点以前的算到前一天
        if (datetime.strptime('20010101 20:59:59.0', '%Y%m%d %H:%M:%S.%f') <
                datetime.strptime('20010101' + ' ' + tick.time,
                                  '%Y%m%d %H:%M:%S.%f') < datetime.strptime(
                                      '20010101 23:30:01.0',
                                      '%Y%m%d %H:%M:%S.%f')):
            tick.datetime = tick.datetime + timedelta(days=-1)
            flt = {'datetime': tick.datetime}
            collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
        elif (datetime.strptime('20010101 8:59:59.0', '%Y%m%d %H:%M:%S.%f') <
              datetime.strptime('20010101' + ' ' + tick.time,
                                '%Y%m%d %H:%M:%S.%f') <
              datetime.strptime('20010101 15:00:01.0', '%Y%m%d %H:%M:%S.%f')):
            flt = {'datetime': tick.datetime}
            collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
        else:
            #把不在开盘时间内的去掉
            print u'抛弃数据:%s ' % tick.time
    csv_file.close()

    print u'插入完毕,耗时:%s' % (time() - start)
Beispiel #26
0
 def __init__(self):
     """Constructor"""
     host, port = loadMongoSetting()
     
     self.dbClient = pymongo.MongoClient(host, port)
     self.datayesClient = DatayesClient()