def loadTbPlusCsv(fileName, dbName, symbol):
"""将TB极速版导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.date = str(d[0])
tempstr=str(round(float(d[1])*10000)).split(".")[0].zfill(4)
bar.time = tempstr[:2]+":"+tempstr[2:4]+":00"
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = d[6]
bar.openInterest = d[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print(bar.date, bar.time)
print('插入完毕，耗时：%s' % (time()-start))
def loadMcCsv(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
import csv
start = time()
print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.DictReader(open(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d['Open'])
bar.high = float(d['High'])
bar.low = float(d['Low'])
bar.close = float(d['Close'])
bar.date = datetime.strptime(d['Date'], '%Y-%m-%d').strftime('%Y%m%d')
bar.time = d['Time']
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = d['TotalVolume']
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print(bar.date, bar.time)
print('插入完毕，耗时：%s' % (time()-start))
def loadOKEXCsv(fileName, dbName, symbol):
"""将OKEX导出的csv格式的历史分钟数据插入到Mongo数据库中"""
start = time()
print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(open(fileName,"r"))
for d in reader:
if len(d[1]) > 10:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.datetime = datetime.strptime(d[1], '%Y-%m-%d %H:%M:%S')
bar.date = bar.datetime.date().strftime('%Y%m%d')
bar.time = bar.datetime.time().strftime('%H:%M:%S')
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.volume = float(d[6])
bar.tobtcvolume = float(d[7])
def loadDayTxt(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
import csv
import pandas as pd
import pymongo
from cyvn.trader.vtObject import VtBarData
getcvs()
start = time()
print (u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient('149.28.56.155',
27017,
username = '******',
password = '******',
connectTimeoutMS=500)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.DictReader(open(fileName, 'r',))
rows = [row for row in reader]
for d in rows[-200:-1]:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.exchange = 'SHFE'
bar.gatewayName = 'CTP'
bar.open = float(d['open'])/10000
bar.high = float(d['high'])/10000
bar.low = float(d['low'])/10000
bar.close = float(d['close'])/10000
bar.date = datetime.strptime(d['date'], '%Y%m%d').strftime('%Y%m%d')
bar.time = "15:00"
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d['volume']
bar.openInterest = d['open_interest']
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print(bar.date)
print (u'插入完毕，耗时：%s' % (time()-start))
def downloadEquityDailyBarts(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print('开始下载%s日行情' %symbol)
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
import tushare as ts
if last:
start = last['date'][:4]+'-'+last['date'][4:6]+'-'+last['date'][6:]
data = ts.get_k_data(symbol,start)
if not data.empty:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for index, d in data.iterrows():
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.open = d.get('open')
bar.high = d.get('high')
bar.low = d.get('low')
bar.close = d.get('close')
bar.date = d.get('date').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('volume')
except KeyError:
print(d)
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print('%s下载完成' %symbol)
else:
print('找不到合约%s' %symbol)
def loadTdxLc1(fileName, dbName, symbol):
"""将通达信导出的lc1格式的历史分钟数据插入到Mongo数据库中"""
start = time()
print('开始读取通达信Lc1文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
#读取二进制文件
ofile=open(fileName,'rb')
buf=ofile.read()
ofile.close()
num=len(buf)
no=num/32
b=0
e=32
dl = []
for i in range(no):
a=unpack('hhfffffii',buf[b:e])
b=b+32
e=e+32
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = a[2]
bar.high = a[3]
bar.low = a[4]
bar.close = a[5]
bar.date = str(int(a[0]/2048)+2004)+str(int(a[0]%2048/100)).zfill(2)+str(a[0]%2048%100).zfill(2)
bar.time = str(int(a[1]/60)).zfill(2)+':'+str(a[1]%60).zfill(2)+':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = a[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print('插入完毕，耗时：%s' % (time()-start))
def loadCsv(filename):
""""""
symbol = filename.split('.')[0]
mc = MongoClient()
db = mc[DAILY_DB_NAME]
collection = db[symbol]
with open(filename) as f:
r = DictReader(f)
for d in r:
bar = VtBarData()
bar.datetime = datetime.strptime(d['date'], '%Y/%m/%d')
bar.vtSymbol = symbol
bar.open = float(d['open'])
bar.high = float(d['high'])
bar.low = float(d['low'])
bar.close = float(d['close'])
bar.volume = int(d['volume'])
collection.insert(bar.__dict__)
def loadRqCsv(fileName, dbName, symbol):
"""将RQ导出的csv格式的历史分钟数据插入到Mongo数据库中"""
start = time()
print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'],
username = globalSetting['mongoUser'],
password = globalSetting['mongoPwd'],
connectTimeoutMS=500)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.DictReader(open(fileName, 'r', ))
rows = [row for row in reader]
for d in rows:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.datetime = datetime.strptime(d['datetime'], '%Y-%m-%d %H:%M:%S')
bar.date = bar.datetime.date().strftime('%Y%m%d')
bar.time = bar.datetime.time().strftime('%H:%M:%S')
bar.exchange = 'SHFE'
bar.gatewayName = 'CTP'
bar.open = float(d['open'])
bar.high = float(d['high'])
bar.low = float(d['low'])
bar.close = float(d['close'])
bar.volume = float(d['volume'])
bar.openInterest = float(d['open_interest'])
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print(('%s \t %s' % (bar.date, bar.time)))
print('插入完毕，耗时：%s' % (time()-start))
def loadTdxCsv(fileName, dbName, symbol):
"""将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
date_correct = ""
print('开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'], globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[1])
bar.high = float(d[2])
bar.low = float(d[3])
bar.close = float(d[4])
#通达信的夜盘时间按照新的一天计算，此处将其按照当天日期统计，方便后续查阅
date_temp,time_temp = d[0].strip(' ').replace('\xef\xbb\xbf','').split('-',1)
if time_temp == '15:00':
date_correct = date_temp
if time_temp[:2] == "21" or time_temp[:2] == "22" or time_temp[:2] == "23":
date_temp = date_correct
bar.date = datetime.strptime(date_temp, '%Y/%m/%d').strftime('%Y%m%d')
bar.time = time_temp[:2]+':'+time_temp[3:5]+':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = d[5]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print('插入完毕，耗时：%s' % (time()-start))
def handleRecorderDay(self, event):
"""从数据库中读取Bar数据，startDate是datetime对象"""
oi = {}
for contact_ in self.barSymbolSet:
time_now = datetime.now()
if datetime.today().weekday() == 0:
#周一接上周五的夜盘
time_yes = time_now + timedelta(-3)
else:
time_yes = time_now + timedelta(-1)
startDate = datetime(time_yes.year, time_yes.month,time_yes.day,21) #前一天 9点
d = {'datetime':{'$gte':startDate}}
barData = self.mainEngine.dbQuery(MINUTE_15_DB_NAME, contact_, d, 'datetime')
day_bar =None
for bar in barData:
# 尚未创建对象
if not day_bar:
day_bar = VtBarData()
day_bar.vtSymbol = bar['vtSymbol']
day_bar.symbol = bar['symbol']
day_bar.exchange = bar['exchange']
day_bar.open = bar['open']
day_bar.high = bar['high']
day_bar.low = bar['low']
# 累加老K线
else:
day_bar.high = max(day_bar.high, bar['high'])
day_bar.low = min(day_bar.low, bar['low'])
# 通用部分
day_bar.close = bar['close']
day_bar.datetime = bar['datetime']
day_bar.openInterest = bar['openInterest']
day_bar.volume += int(bar['volume'])
if day_bar:
day_bar.datetime = datetime(time_now.year, time_now.month,time_now.day)
day_bar.date = day_bar.datetime.strftime('%Y%m%d')
day_bar.time = day_bar.datetime.strftime('%H:%M:%S')
self.mainEngine.dbInsert(DAILY_DB_NAME, contact_, day_bar.__dict__)
if contact_ in self.activeSymbolDict:
activeSymbol = self.activeSymbolDict[contact_]
self.mainEngine.dbInsert(DAILY_DB_NAME, activeSymbol, day_bar.__dict__)
# 写入持仓量数据
oi[day_bar.symbol] = day_bar.openInterest
# 保存持仓量数据
filename = 'openInterest.json'
json_data = {'oi': oi}
d1 = json.dumps(json_data, sort_keys=True, indent=4)
f = open(os.path.join(os.getcwd(), filename), 'w')
f.write(d1)
f.close()
def convertBar(collectionName, tickDb, barDb):
"""处理数据"""
# 处理夜盘收盘的分钟线问题
nonum_name = filter(str.isalpha, str(collectionName))
today = datetime.now()
yesterday = today - timedelta(1) # 昨天
startTime = None
"""夜盘处理（凌晨时间处理）"""
if today.time() > NIGHT_END and today.time() < time(11, 0):
if nonum_name in NIGHT_MARKET_DICT['SHF1']:
# 23:00结束合约
start = yesterday
startTime = start.replace(hour=22,
minute=59,
second=0,
microsecond=0)
elif nonum_name in NIGHT_MARKET_DICT['SHF2']:
# 01:00结束合约
start = today # 今天
startTime = start.replace(hour=0,
minute=59,
second=0,
microsecond=0)
elif nonum_name in NIGHT_MARKET_DICT['SHF3']:
# 02:30结束合约
start = today # 今天
startTime = start.replace(hour=2,
minute=29,
second=0,
microsecond=0)
elif (nonum_name in NIGHT_MARKET_DICT['DCE']
or nonum_name in NIGHT_MARKET_DICT['CZC']):
# 23:30结束合约
start = yesterday # 昨天
startTime = start.replace(hour=23,
minute=29,
second=0,
microsecond=0)
else:
# 其他合约夜盘不做处理
startTime = None
"""白天收盘处理"""
if today.time() > AFTERNOON_END and today.time() < NIGHT_START:
if nonum_name in MARKET_BOND:
start = today # 今天
startTime = start.replace(hour=15,
minute=14,
second=0,
microsecond=0)
else:
start = today # 今天
startTime = start.replace(hour=14,
minute=59,
second=0,
microsecond=0)
if startTime:
mc = MongoClient('localhost', 27017) # 创建MongoClient
cl = mc[tickDb][collectionName] # 获取tick数据集合
d = {'datetime': {'$gte': startTime}} # 只过滤从start开始的数据
cx = cl.find(d).sort('datetime') # 获取数据指针
lastTime = startTime - timedelta(minutes=1)
lastTick = None
# 获取上一个tick
d = {'datetime': {'$gte': lastTime, '$lt': startTime}}
lastData = cl.find(d).sort('datetime') # 获取数据指针
for i in lastData:
lastTick = i
# 遍历数据
bar = None
for tick in cx:
# 转换tick数据为bar数据
if not bar:
bar = VtBarData()
#dt = data['datetime'].time()
bar.vtSymbol = tick['vtSymbol']
bar.symbol = tick['symbol']
bar.exchange = tick['exchange']
bar.open = tick['lastPrice']
bar.high = tick['lastPrice']
bar.low = tick['lastPrice']
else:
bar.high = max(bar.high, tick['lastPrice'])
bar.low = min(bar.low, tick['lastPrice'])
# 通用更新部分
bar.close = tick['lastPrice']
bar.datetime = tick['datetime']
bar.openInterest = tick['openInterest']
if lastTick:
bar.volume += (tick['volume'] - lastTick['volume']
) # 当前K线内的成交量
# 缓存Tick
lastTick = tick
# 转换bar时间戳,考虑可能节假日启动程序，但是没有收盘tick
if bar:
bar.datetime = startTime # 使用目标的时间日期
bar.date = bar.datetime.strftime('%Y%m%d')
bar.time = bar.datetime.strftime('%H:%M:%S.%f')
# 更新到bar数据库
flt = {'datetime': bar.datetime}
clBar = mc[barDb][collectionName]
clBar.update_one(flt, {'$set': bar.__dict__}, upsert=True)
# 重置bar
bar = None
print(u'分钟线处理完成，数据库：%s, 集合：%s' % (barDb, collectionName))