def loadMcCsv(fileName, dbName, symbol): """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中""" import csv start = time() print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)) # 锁定集合,并创建索引 client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort']) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) # 读取数据和插入到数据库 reader = csv.DictReader(open(fileName, 'r')) for d in reader: bar = VtBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.open = float(d['Open']) bar.high = float(d['High']) bar.low = float(d['Low']) bar.close = float(d['Close']) bar.date = datetime.strptime(d['Date'], '%Y-%m-%d').strftime('%Y%m%d') bar.time = d['Time'] bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S') bar.volume = d['TotalVolume'] flt = {'datetime': bar.datetime} collection.update_one(flt, {'$set':bar.__dict__}, upsert=True) print(bar.date, bar.time) print('插入完毕,耗时:%s' % (time()-start))
def loadOKEXCsv(fileName, dbName, symbol): """将OKEX导出的csv格式的历史分钟数据插入到Mongo数据库中""" start = time() print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)) # 锁定集合,并创建索引 client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort']) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) # 读取数据和插入到数据库 reader = csv.reader(open(fileName,"r")) for d in reader: if len(d[1]) > 10: bar = VtBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.datetime = datetime.strptime(d[1], '%Y-%m-%d %H:%M:%S') bar.date = bar.datetime.date().strftime('%Y%m%d') bar.time = bar.datetime.time().strftime('%H:%M:%S') bar.open = float(d[2]) bar.high = float(d[3]) bar.low = float(d[4]) bar.close = float(d[5]) bar.volume = float(d[6]) bar.tobtcvolume = float(d[7])
def loadTbPlusCsv(fileName, dbName, symbol): """将TB极速版导出的csv格式的历史分钟数据插入到Mongo数据库中""" import csv start = time() print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)) # 锁定集合,并创建索引 client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort']) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) # 读取数据和插入到数据库 reader = csv.reader(file(fileName, 'r')) for d in reader: bar = VtBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.open = float(d[2]) bar.high = float(d[3]) bar.low = float(d[4]) bar.close = float(d[5]) bar.date = str(d[0]) tempstr=str(round(float(d[1])*10000)).split(".")[0].zfill(4) bar.time = tempstr[:2]+":"+tempstr[2:4]+":00" bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S') bar.volume = d[6] bar.openInterest = d[7] flt = {'datetime': bar.datetime} collection.update_one(flt, {'$set':bar.__dict__}, upsert=True) print(bar.date, bar.time) print('插入完毕,耗时:%s' % (time()-start))
def loadDayTxt(fileName, dbName, symbol): """将Multicharts导出的csv格式的历史数据插入到Mongo数据库中""" import csv import pandas as pd import pymongo from cyvn.trader.vtObject import VtBarData getcvs() start = time() print (u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)) # 锁定集合,并创建索引 client = pymongo.MongoClient('149.28.56.155', 27017, username = '******', password = '******', connectTimeoutMS=500) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) # 读取数据和插入到数据库 reader = csv.DictReader(open(fileName, 'r',)) rows = [row for row in reader] for d in rows[-200:-1]: bar = VtBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.exchange = 'SHFE' bar.gatewayName = 'CTP' bar.open = float(d['open'])/10000 bar.high = float(d['high'])/10000 bar.low = float(d['low'])/10000 bar.close = float(d['close'])/10000 bar.date = datetime.strptime(d['date'], '%Y%m%d').strftime('%Y%m%d') bar.time = "15:00" bar.datetime = datetime.strptime(bar.date, '%Y%m%d') bar.volume = d['volume'] bar.openInterest = d['open_interest'] flt = {'datetime': bar.datetime} collection.update_one(flt, {'$set':bar.__dict__}, upsert=True) print(bar.date) print (u'插入完毕,耗时:%s' % (time()-start))
def downloadEquityDailyBarts(self, symbol): """ 下载股票的日行情,symbol是股票代码 """ print('开始下载%s日行情' %symbol) # 查询数据库中已有数据的最后日期 cl = self.dbClient[DAILY_DB_NAME][symbol] cx = cl.find(sort=[('datetime', pymongo.DESCENDING)]) if cx.count(): last = cx[0] else: last = '' # 开始下载数据 import tushare as ts if last: start = last['date'][:4]+'-'+last['date'][4:6]+'-'+last['date'][6:] data = ts.get_k_data(symbol,start) if not data.empty: # 创建datetime索引 self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)], unique=True) for index, d in data.iterrows(): bar = VtBarData() bar.vtSymbol = symbol bar.symbol = symbol try: bar.open = d.get('open') bar.high = d.get('high') bar.low = d.get('low') bar.close = d.get('close') bar.date = d.get('date').replace('-', '') bar.time = '' bar.datetime = datetime.strptime(bar.date, '%Y%m%d') bar.volume = d.get('volume') except KeyError: print(d) flt = {'datetime': bar.datetime} self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True) print('%s下载完成' %symbol) else: print('找不到合约%s' %symbol)
def loadTdxLc1(fileName, dbName, symbol): """将通达信导出的lc1格式的历史分钟数据插入到Mongo数据库中""" start = time() print('开始读取通达信Lc1文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)) # 锁定集合,并创建索引 client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort']) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) #读取二进制文件 ofile=open(fileName,'rb') buf=ofile.read() ofile.close() num=len(buf) no=num/32 b=0 e=32 dl = [] for i in range(no): a=unpack('hhfffffii',buf[b:e]) b=b+32 e=e+32 bar = VtBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.open = a[2] bar.high = a[3] bar.low = a[4] bar.close = a[5] bar.date = str(int(a[0]/2048)+2004)+str(int(a[0]%2048/100)).zfill(2)+str(a[0]%2048%100).zfill(2) bar.time = str(int(a[1]/60)).zfill(2)+':'+str(a[1]%60).zfill(2)+':00' bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S') bar.volume = a[7] flt = {'datetime': bar.datetime} collection.update_one(flt, {'$set':bar.__dict__}, upsert=True) print('插入完毕,耗时:%s' % (time()-start))
def loadCsv(filename): """""" symbol = filename.split('.')[0] mc = MongoClient() db = mc[DAILY_DB_NAME] collection = db[symbol] with open(filename) as f: r = DictReader(f) for d in r: bar = VtBarData() bar.datetime = datetime.strptime(d['date'], '%Y/%m/%d') bar.vtSymbol = symbol bar.open = float(d['open']) bar.high = float(d['high']) bar.low = float(d['low']) bar.close = float(d['close']) bar.volume = int(d['volume']) collection.insert(bar.__dict__)
def loadRqCsv(fileName, dbName, symbol): """将RQ导出的csv格式的历史分钟数据插入到Mongo数据库中""" start = time() print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)) # 锁定集合,并创建索引 client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'], username = globalSetting['mongoUser'], password = globalSetting['mongoPwd'], connectTimeoutMS=500) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) # 读取数据和插入到数据库 reader = csv.DictReader(open(fileName, 'r', )) rows = [row for row in reader] for d in rows: bar = VtBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.datetime = datetime.strptime(d['datetime'], '%Y-%m-%d %H:%M:%S') bar.date = bar.datetime.date().strftime('%Y%m%d') bar.time = bar.datetime.time().strftime('%H:%M:%S') bar.exchange = 'SHFE' bar.gatewayName = 'CTP' bar.open = float(d['open']) bar.high = float(d['high']) bar.low = float(d['low']) bar.close = float(d['close']) bar.volume = float(d['volume']) bar.openInterest = float(d['open_interest']) flt = {'datetime': bar.datetime} collection.update_one(flt, {'$set':bar.__dict__}, upsert=True) print(('%s \t %s' % (bar.date, bar.time))) print('插入完毕,耗时:%s' % (time()-start))
def loadTdxCsv(fileName, dbName, symbol): """将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中""" import csv start = time() date_correct = "" print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)) # 锁定集合,并创建索引 client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort']) collection = client[dbName][symbol] collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True) # 读取数据和插入到数据库 reader = csv.reader(file(fileName, 'r')) for d in reader: bar = VtBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.open = float(d[1]) bar.high = float(d[2]) bar.low = float(d[3]) bar.close = float(d[4]) #通达信的夜盘时间按照新的一天计算,此处将其按照当天日期统计,方便后续查阅 date_temp,time_temp = d[0].strip(' ').replace('\xef\xbb\xbf','').split('-',1) if time_temp == '15:00': date_correct = date_temp if time_temp[:2] == "21" or time_temp[:2] == "22" or time_temp[:2] == "23": date_temp = date_correct bar.date = datetime.strptime(date_temp, '%Y/%m/%d').strftime('%Y%m%d') bar.time = time_temp[:2]+':'+time_temp[3:5]+':00' bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S') bar.volume = d[5] flt = {'datetime': bar.datetime} collection.update_one(flt, {'$set':bar.__dict__}, upsert=True) print('插入完毕,耗时:%s' % (time()-start))