def loadTdxCsv(fileName, dbName, symbol):
"""将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
print(u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
# 锁定集合，并创建索引
host, port, logging = loadMongoSetting()
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.date = datetime.strptime(d[0], '%Y/%m/%d').strftime('%Y%m%d')
bar.time = d[1][:2]+':'+d[1][2:4]+':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = d[6]
bar.openInterest = d[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print(bar.date, bar.time)
print(u'插入完毕，耗时：%s' % (time()-start))
def loadMcCsv(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
import csv
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol)
# 锁定集合，并创建索引
host, port = loadMongoSetting()
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.DictReader(file(fileName, 'r'))
for d in reader:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d['Open'])
bar.high = float(d['High'])
bar.low = float(d['Low'])
bar.close = float(d['Close'])
bar.date = datetime.strptime(d['Date'], '%Y/%m/%d').strftime('%Y%m%d')
bar.time = d['Time']
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
bar.volume = d['TotalVolume']
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print bar.date, bar.time
print u'插入完毕，耗时：%s' % (time()-start)
def __init__(self):
"""Constructor"""
host, port, logging = loadMongoSetting()
print(host)
self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()
def loadMcCsv(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
import csv
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
# 锁定集合，并创建索引
host, port = loadMongoSetting()
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.DictReader(file(fileName, 'r'))
for d in reader:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d['Open'])
bar.high = float(d['High'])
bar.low = float(d['Low'])
bar.close = float(d['Close'])
bar.date = datetime.strptime(d['Date'], '%Y/%m/%d').strftime('%Y%m%d')
bar.time = d['Time']
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d['TotalVolume']
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print bar.date, bar.time
print u'插入完毕，耗时：%s' % (time() - start)
def loadHistoryData(self):
"""载入历史数据"""
"""load historical data"""
host, port = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
collection = self.dbClient[self.dbName][self.symbol]
self.output("Start loading historical data")
# 首先根据回测模式，确认要使用的数据类
# Choose data type based on backtest mode
if self.mode == self.BAR_MODE:
dataClass = CtaBarData
else:
dataClass = CtaTickData
# 载入初始化需要用的数据
# Load initialised data
# $gte means "greater and equal to"
# $lt means "less than"
flt = {
'datetime': {
'$gte': self.dataStartDate,
'$lt': self.strategyStartDate
}
}
initCursor = collection.find(flt)
# 将数据从查询指针中读取出，并生成列表
# Read data from cursor, generate a list
self.initData = [] # Empty "initData" list
for d in initCursor:
data = dataClass()
data.__dict__ = d
self.initData.append(data)
# 载入回测数据
# Load backtest data (exclude initialised data)
if not self.dataEndDate:
# If "End Date" is not set, retreat data up to today
flt = {'datetime': {'$gte': self.strategyStartDate}}
else:
flt = {
'datetime': {
'$gte': self.strategyStartDate,
'$lte': self.dataEndDate
}
}
self.dbCursor = collection.find(flt)
self.output("Data loading complete, data volumn: %s" %
(initCursor.count() + self.dbCursor.count()))
def loadHistoryData(self):
"""载入历史数据"""
"""load historical data"""
host, port, logging = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
collection = self.dbClient[self.dbName][self.symbol]
# Load historical data of information symbols, construct a dictionary of Database
# Values of dictionary are mongo.Client.
info_collection = {}
if self.MultiOn is True:
for DBname, symbol in self.info_symbols:
info_collection[DBname + " " +
symbol] = self.dbClient[DBname][symbol]
self.output("Start loading historical data")
# 首先根据回测模式，确认要使用的数据类
# Choose data type based on backtest mode
if self.mode == self.BAR_MODE:
self.dataClass = CtaBarData
self.func = self.newBar
else:
self.dataClass = CtaTickData
self.func = self.newTick
# Load initializing data
self.loadInitData(collection, inf=info_collection)
# 载入回测数据
# Load backtest data (exclude initializing data)
if not self.dataEndDate:
# If "End Date" is not set, retreat data up to today
flt = {'datetime': {'$gte': self.strategyStartDate}}
else:
flt = {
'datetime': {
'$gte': self.strategyStartDate,
'$lte': self.dataEndDate
}
}
self.dbCursor = collection.find(flt)
if self.MultiOn is True:
for db in info_collection:
self.InfoCursor[db] = info_collection[db].find(flt)
self.output(
"Data loading completed, data volumn: %s" % (self.initCursor.count() + self.dbCursor.count() + \
sum([i.count() for i in self.InfoCursor.values()])))
else:
self.output("Data loading completed, data volumn: %s" %
(self.initCursor.count() + self.dbCursor.count()))
def loadTBCsv(fileName, dbName, symbol):
"""将TradeBlazer导出的csv格式的历史数据插入到Mongo数据库中
数据样本：
//时间,开盘价,最高价,最低价,收盘价,成交量,持仓量
2017/04/05 09:00,3200,3240,3173,3187,312690,2453850
20150105,0.0911,2572,2574,2571,2573,27744,2570474
"""
import csv
start = time()
print(u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
host, port, logging = loadMongoSetting()
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
if len(d[0]) == 8:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
# bar.datetime = datetime.strptime(d[0], '%Y/%m/%d %H:%M')
# bar.date = bar.datetime.date().strftime('%Y%m%d')
# bar.time = bar.datetime.time().strftime('%H:%M:%S')
deltaMinutes = float(d[1]) * 10000
bar.datetime = datetime.strptime(d[0], '%Y%m%d') + timedelta(
hours=int(float(d[1]) * 100),
minutes=int(float(d[1]) * 10000) % 100)
bar.date = bar.datetime.date().strftime('%Y%m%d')
bar.time = bar.datetime.time().strftime('%H:%M:%S')
bar.open = float(d[1 + 1])
bar.high = float(d[2 + 1])
bar.low = float(d[3 + 1])
bar.close = float(d[4 + 1])
bar.volume = float(d[5 + 1])
bar.openInterest = float(d[6 + 1])
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print('%s \t %s' % (bar.date, bar.time))
print(u'插入完毕，耗时：%s' % (time() - start))
def loadCsv(filename, dbName, symbol, exch):
start = time()
#print u'Start'
host, port = loadMongoSetting()
#print host, port
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
reader = csv.reader(file(filename, 'r'))
first_line = True
for d in reader:
try:
if first_line or d[6] == '0' or d[7] == '0' or d[12] == '0' or d[
13] == '0':
first_line = False
continue
except IndexError:
break
tick = CtaTickData()
tick.vtSymbol = symbol
tick.symbol = symbol
tick.exchange = exch
tick.lastPrice = float(d[2])
tick.volume = float(d[3])
tick.openInterest = float(d[5])
tick.date = datetime.strptime(d[0], '%Y-%m-%d').strftime('%Y%m%d')
tick.time = d[1]
tick.datetime = datetime.strptime(tick.date + ' ' + tick.time,
'%Y%m%d %H:%M:%S')
tick.bidPrice1 = float(d[12])
#tick.bidPrice2=float(d[14])
#tick.bidPrice3=float(d[16])
tick.askPrice1 = float(d[6])
#tick.askPrice2=float(d[8])
#tick.askPrice3=float(d[10])
tick.bidVolume1 = float(d[13])
#tick.bidVolume2=float(d[15])
#tick.bidVolume3=float(d[17])
tick.askVolume1 = float(d[7])
#tick.askVolume2=float(d[9])
#tick.askVolume3=float(d[11])
flt = {'datetime': tick.datetime}
collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
print symbol, tick.date, tick.time
print u'OK', filename
def loadCsv(filename, dbName, symbol):
#print u'Start'
host, port, logging = loadMongoSetting()
#print host, port
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
reader = csv.reader(file(filename, 'r'))
first_line = True
lastdatetime = None
temp_list = []
for d in reader:
try:
if first_line or d[6] == '0' or d[7] == '0' or d[12] == '0' or d[
13] == '0':
first_line = False
continue
except IndexError:
break
temp = {'lastPrice': 0.0, 'datetime': None}
date = datetime.strptime(d[0], '%Y-%m-%d').strftime('%Y%m%d')
time = datetime.strptime(d[1], '%H:%M:%S').strftime('%H:%M:%S')
temp['datetime'] = datetime.strptime(date + ' ' + time,
'%Y%m%d %H:%M:%S')
if lastdatetime != temp['datetime']:
temp['lastPrice'] = float(d[2])
temp_list.append(temp)
lastdatetime = temp['datetime']
else:
temp1 = temp_list.pop()
t = temp1['lastPrice']
temp1['lastPrice'] = (float(t) + float(d[2])) / 2
temp_list.append(temp1)
lastdatetime = temp1['datetime']
#print temp['datetime']
try:
collection.insert(temp_list)
print u'OK', filename
except:
for i in temp_list:
try:
collection.insert([i])
print u'OK', i['datetime']
except:
print u'Fail', i['datetime']
continue
def loadHistoryData(self):
"""载入历史数据"""
host, port = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
self.output(u'开始载入数据')
for symbol in self.symbols:
self.output(u"载入历史数据" + str(symbol))
collection = self.dbClient[self.dbName][symbol]
# 载入回测数据
if not self.dataEndDate:
flt = {'datetime': {'$gte': self.strategyStartDate}} # 数据过滤条件
else:
flt = {
'datetime': {
'$gte': self.strategyStartDate,
'$lte': self.dataEndDate
}
}
dbCursor = collection.find(flt)
initData = []
for d in dbCursor:
initData.append(d)
if self.dataframe.empty:
self.dataframe = pd.DataFrame(initData)
else:
self.dataframe = pd.concat(
[self.dataframe, pd.DataFrame(initData)])
#这里只管喂就好了，初始化的逻辑交给策略
df = self.dataframe.drop_duplicates()
del df['_id']
df = to_transed_eve(df)
df = df.sort_values('datetime')
index = pd.Index(np.arange(df.count()[0]))
df.index = index
df[['open', 'close', 'low',
'high']] = df[['open', 'close', 'low', 'high']].applymap(float)
df[['volume']] = df[['volume']].applymap(int)
self.dataframe = df
self.output(u'载入完成，数据量：%s' % (self.dataframe.count()[0]))
def loadHistoryData(self):
"""载入历史数据"""
"""load historical data"""
host, port, logging = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
collection = self.dbClient[self.dbName][self.symbol]
# Load historical data of information symbols, construct a dictionary of Database
# Values of dictionary are mongo.Client.
info_collection = {}
if self.MultiOn is True:
for DBname, symbol in self.info_symbols:
info_collection[DBname + " " + symbol] = self.dbClient[DBname][symbol]
self.output("Start loading historical data")
# 首先根据回测模式，确认要使用的数据类
# Choose data type based on backtest mode
if self.mode == self.BAR_MODE:
self.dataClass = CtaBarData
self.func = self.newBar
else:
self.dataClass = CtaTickData
self.func = self.newTick
# Load initializing data
self.loadInitData(collection, inf=info_collection)
# 载入回测数据
# Load backtest data (exclude initializing data)
if not self.dataEndDate:
# If "End Date" is not set, retreat data up to today
flt = {'datetime': {'$gte': self.strategyStartDate}}
else:
flt = {'datetime': {'$gte': self.strategyStartDate,
'$lte': self.dataEndDate}}
self.dbCursor = collection.find(flt)
if self.MultiOn is True:
for db in info_collection:
self.InfoCursor[db] = info_collection[db].find(flt)
self.output(
"Data loading completed, data volumn: %s" % (self.initCursor.count() + self.dbCursor.count() + \
sum([i.count() for i in self.InfoCursor.values()])))
else:
self.output("Data loading completed, data volumn: %s" % (self.initCursor.count() + self.dbCursor.count()))
def dbConnect(self):
"""连接MongoDB数据库"""
if not self.dbClient:
# 读取MongoDB的设置
host, port = loadMongoSetting()
try:
# 设置MongoDB操作的超时时间为0.5秒
self.dbClient = MongoClient(host, port, serverSelectionTimeoutMS=500)
# 调用server_info查询服务器状态，防止服务器异常并未连接成功
self.dbClient.server_info()
self.writeLog(u'MongoDB连接成功')
except ConnectionFailure:
self.writeLog(u'MongoDB连接失败')
def loadMinuteTxt(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
import csv
import pandas as pd
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
# 锁定集合，并创建索引
host, port, logging = loadMongoSetting()
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
lines = open(fileName, 'r').readlines()
lines = [line.replace('\t', ',') for line in lines]
newfile = open('swap.txt', 'w')
newfile.writelines('Date,Time,Open,High,Low,Close,Vol,Val\r\n')
newfile.writelines(lines)
newfile.close()
reader = csv.DictReader(file(
'swap.txt',
'r',
))
for d in reader:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d['Open'])
bar.high = float(d['High'])
bar.low = float(d['Low'])
bar.close = float(d['Close'])
bar.date = datetime.strptime(d['Date'], '%Y/%m/%d').strftime('%Y%m%d')
bar.time = d['Time']
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M')
bar.volume = d['Vol']
bar.openInterest = d['Val']
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print bar.date, bar.time
print u'插入完毕，耗时：%s' % (time() - start)
def loadHistoryData(self):
"""载入历史数据"""
host, port, logging = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
collection = self.dbClient[self.dbName][self.symbol]
self.output(u'开始载入数据')
# 首先根据回测模式，确认要使用的数据类
if self.mode == self.BAR_MODE:
dataClass = CtaBarData
func = self.newBar
else:
dataClass = CtaTickData
func = self.newTick
# 载入初始化需要用的数据
flt = {
'datetime': {
'$gte': self.dataStartDate,
'$lt': self.strategyStartDate
}
}
initCursor = collection.find(flt)
# 将数据从查询指针中读取出，并生成列表
self.initData = [] # 清空initData列表
for d in initCursor:
data = dataClass()
data.__dict__ = d
self.initData.append(data)
# 载入回测数据
if not self.dataEndDate:
flt = {'datetime': {'$gte': self.strategyStartDate}} # 数据过滤条件
else:
flt = {
'datetime': {
'$gte': self.strategyStartDate,
'$lte': self.dataEndDate
}
}
self.dbCursor = collection.find(flt)
self.output(u'载入完成，数据量：%s' %
(initCursor.count() + self.dbCursor.count()))
def loadTBCsv(fileName, dbName, symbol):
"""将TradeBlazer导出的csv格式的历史数据插入到Mongo数据库中
数据样本：
//时间,开盘价,最高价,最低价,收盘价,成交量,持仓量
2017/04/05 09:00,3200,3240,3173,3187,312690,2453850
"""
import csv
'''2017052500 Add by hetajen begin'''
start = time.time()
'''2017052500 Add by hetajen end'''
print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
# 锁定集合，并创建索引
host, port, logging = loadMongoSetting()
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
if len(d[0]) > 10:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
'''2017052500 Add by hetajen begin'''
bar.datetime = datetime.datetime.strptime(d[0], '%Y/%m/%d %H:%M')
'''2017052500 Add by hetajen end'''
bar.date = bar.datetime.date().strftime('%Y%m%d')
bar.time = bar.datetime.time().strftime('%H:%M:%S')
bar.open = float(d[1])
bar.high = float(d[2])
bar.low = float(d[3])
bar.close = float(d[4])
bar.volume = float(d[5])
bar.openInterest = float(d[6])
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print '%s \t %s' % (bar.date, bar.time)
'''2017052500 Add by hetajen begin'''
print u'插入完毕，耗时：%s' % (time.time() - start)
'''2017052500 Add by hetajen end'''
def dbConnect(self):
"""连接MongoDB数据库"""
# 读取数据库配置的方法，已转移至vtFunction
# # 载入json文件
# fileName = 'Mongo_connect.json'
# fileName = os.getcwd() + '\\mongoConnect\\' + fileName
#
# try:
# f = file(fileName)
# except IOError:
# self.writeLog(u'读取MongoDB连接配置出错，请检查')
# return
#
# # 解析json文件
# setting = json.load(f)
# try:
# IP = str(setting['IP'])
# replicaset = str(setting['replicaset'])
# readPreference = str(setting['readPreference'])
# database = str(setting['db'])
# userID = str(setting['userID'])
# password = str(setting['password'])
#
# except KeyError:
# self.writeLog(u'MongoDB连接配置缺少字段，请检查')
if not self.dbClient:
# 读取MongoDB的设置
host, port, replicaset, readPreference, database, userID, password = loadMongoSetting()
try:
# 设置MongoDB操作的超时时间为0.5秒
# self.dbClient = pymongo.MongoClient(host+':'+str(port), replicaset=replicaset,readPreference=readPreference, serverSelectionTimeoutMS=500)
# db = self.dbClient[database]
# db.authenticate(userID, password)
self.dbClient = pymongo.MongoClient(host, port, serverSelectionTimeoutMS=500)
# 调用server_info查询服务器状态，防止服务器异常并未连接成功
self.dbClient.server_info()
self.writeLog(u"MongoDB连接成功")
except ConnectionFailure:
self.writeLog(u"MongoDB连接失败")
except ValueError:
self.writeLog(u"MongoDB连接配置字段错误，请检查")
def dbConnect(self):
"""连接MongoDB数据库"""
if not self.dbClient:
# 读取MongoDB的设置
host, port = loadMongoSetting()
try:
# 设置MongoDB操作的超时时间为0.5秒
self.dbClient = MongoClient(host,
port,
serverSelectionTimeoutMS=500)
# 调用server_info查询服务器状态，防止服务器异常并未连接成功
self.dbClient.server_info()
self.writeLog(u'MongoDB连接成功')
except ConnectionFailure:
self.writeLog(u'MongoDB连接失败')
def loadTBCsv(fileName, dbName, symbol):
"""将TradeBlazer导出的csv格式的历史数据插入到Mongo数据库中
数据样本：
//时间,开盘价,最高价,最低价,收盘价,成交量,持仓量
2017/04/05 09:00,3200,3240,3173,3187,312690,2453850
"""
import csv
start = time()
print(u'开始读取CSV文件%s中的数据插入到%s的%s中' %(fileName, dbName, symbol))
# 锁定集合，并创建索引
host, port, logging = loadMongoSetting()
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
if len(d[0]) > 10:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.datetime = datetime.strptime(d[0], '%Y/%m/%d %H:%M')
bar.date = bar.datetime.date().strftime('%Y%m%d')
bar.time = bar.datetime.time().strftime('%H:%M:%S')
bar.open = float(d[1])
bar.high = float(d[2])
bar.low = float(d[3])
bar.close = float(d[4])
bar.volume = float(d[5])
bar.openInterest = float(d[6])
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set':bar.__dict__}, upsert=True)
print('%s \t %s' % (bar.date, bar.time))
print(u'插入完毕，耗时：%s' % (time()-start))
def loadMcCsv(fileName, dbName, symbol):
import csv
from time import time
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
# 锁定集合，并创建索引
host, port = loadMongoSetting()
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.DictReader(file(fileName, 'r'))
for d in reader:
tick = CtaTickData()
tick.vtSymbol = symbol
tick.symbol = d['CONTRACTID']
tick.lastPrice = float(d['LASTPX'])
tick.bidPrice1 = float(d['B1'])
tick.askPrice1 = float(d['S1'])
tick.bidVolume1 = float(d['BV1'])
tick.askVolume1 = float(d['SV1'])
tick.volume = float(d['LASTQTY'])
tick.openInterest = float(d['OPENINTS'])
tick.date = float(d['TRADINGDATE'])
tick.datetime = datetime.strptime(d['TDATETIME'][:19],
'%Y-%m-%d %H:%M:%S')
tick.date = datetime.strptime(d['TRADINGDATE'],
'%Y%m%d').strftime('%Y%m%d')
tick.upperLimit = d['RISELIMIT']
tick.lowerLimit = d['FALLLIMIT']
tick.time = d['TDATETIME']
tick.exchange = d['MARKET']
flt = {'datetime': tick.datetime}
collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
print u'正在插入:', tick.date, tick.time, u'的历史数据到mongodb'
print u'插入完毕，耗时：%s' % (time() - start)
def dbConnect(self):
"""连接MongoDB数据库"""
# 载入json文件
if not self.dbClient:
# 读取MongoDB的设置
host, port, replicaset, readPreference, database, userID, password = loadMongoSetting()
try:
self.dbClient = pymongo.MongoClient(host+':'+str(port), replicaset=replicaset,readPreference=readPreference)
db = self.dbClient[database]
db.authenticate(userID, password)
# self.dbClient = MongoClient(host, port)
self.writeCtaLog(u'MongoDB连接成功')
except ConnectionFailure:
self.writeCtaLog(u'MongoDB连接失败')
except ValueError:
self.writeCtaLog(u'MongoDB连接配置字段错误，请检查')
self.mainEngine.dbClient = self.dbClient # 主引擎数据库连接
def dbConnect(self):
"""连接MongoDB数据库"""
# 读取数据库配置的方法，已转移至vtFunction
if not self.dbClient:
# 读取MongoDB的设置
settingFileName = "VT_setting.json"
settingFileName = os.path.dirname(os.getcwd()) + "/" + settingFileName
host, port, replicaset, readPreference, database, userID, password = loadMongoSetting(settingFileName)
try:
# self.dbClient = pymongo.MongoClient(host+':'+str(port), replicaset=replicaset,readPreference=readPreference)
# db = self.dbClient[database]
# db.authenticate(userID, password)
self.dbClient = pymongo.MongoClient(host, port, serverSelectionTimeoutMS=500)
print u'MongoDB连接成功'
except ConnectionFailure:
print u'MongoDB连接失败'
except ValueError:
print u'MongoDB连接配置字段错误，请检查'
def dbConnect(self):
"""连接MongoDB数据库"""
if not self.dbClient:
# 读取MongoDB的设置
host, port, logging = loadMongoSetting()
try:
# 设置MongoDB操作的超时时间为0.5秒
self.dbClient = MongoClient(host, port, connectTimeoutMS=500)
# 调用server_info查询服务器状态，防止服务器异常并未连接成功
self.dbClient.server_info()
self.writeLog(text.DATABASE_CONNECTING_COMPLETED)
# 如果启动日志记录，则注册日志事件监听函数
if logging:
self.eventEngine.register(EVENT_LOG, self.dbLogging)
except ConnectionFailure:
self.writeLog(text.DATABASE_CONNECTING_FAILED)
def dbConnect(self):
"""连接MongoDB数据库"""
if not self.dbClient:
# 读取MongoDB的设置
host, port, logging = loadMongoSetting()
try:
# 设置MongoDB操作的超时时间为0.5秒
self.dbClient = MongoClient(host, port, connectTimeoutMS=500)
# 调用server_info查询服务器状态，防止服务器异常并未连接成功
self.dbClient.server_info()
self.writeLog(text.DATABASE_CONNECTING_COMPLETED)
# 如果启动日志记录，则注册日志事件监听函数
if logging:
self.eventEngine.register(EVENT_LOG, self.dbLogging)
except ConnectionFailure:
self.writeLog(text.DATABASE_CONNECTING_FAILED)
def loadHistoryData(self):
"""载入历史数据"""
host, port, logging = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
collection = self.dbClient[self.dbName][self.symbol]
self.output(u'开始载入数据')
# 首先根据回测模式，确认要使用的数据类
if self.mode == self.BAR_MODE:
dataClass = CtaBarData
func = self.newBar
else:
dataClass = CtaTickData
func = self.newTick
# 载入初始化需要用的数据
flt = {'datetime':{'$gte':self.dataStartDate,
'$lt':self.strategyStartDate}}
initCursor = collection.find(flt)
# 将数据从查询指针中读取出，并生成列表
self.initData = [] # 清空initData列表
for d in initCursor:
data = dataClass()
data.__dict__ = d
self.initData.append(data)
# 载入回测数据
if not self.dataEndDate:
flt = {'datetime':{'$gte':self.strategyStartDate}} # 数据过滤条件
else:
flt = {'datetime':{'$gte':self.strategyStartDate,
'$lte':self.dataEndDate}}
self.dbCursor = collection.find(flt)
self.output(u'载入完成，数据量：%s' %(initCursor.count() + self.dbCursor.count()))
def loadMcCsvTick(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
import csv
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
"""
csv_file = open(fileName, 'r')
reader = csv.DictReader(csv_file)
for d in reader:
print float(d['\xc9\xea\xc2\xf2\xbc\xdb\xd2\xbb'])
csv_file.close()
"""
# 锁定集合，并创建索引
host, port = loadMongoSetting()
client = pymongo.MongoClient(host, port)
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
csv_file = open(fileName, 'r')
reader = csv.DictReader(csv_file)
for d in reader:
tick = CtaTickData()
tick.vtSymbol = symbol
tick.symbol = symbol
tick.askPrice1 = float(d['\xc9\xea\xc2\xf2\xbc\xdb\xd2\xbb']) #申买价一
tick.askVolume1 = int(d['\xc9\xea\xc2\xf2\xc1\xbf\xd2\xbb']) #申买量一
tick.bidPrice1 = float(d['\xc9\xea\xc2\xf4\xbc\xdb\xd2\xbb']) #申卖价一
tick.bidVolume1 = int(d['\xc9\xea\xc2\xf4\xc1\xbf\xd2\xbb']) #申卖量一
tick.lastPrice = float(d['\xd7\xee\xd0\xc2\xbc\xdb']) #最新价
tick.upperLimit = float(d['\xd5\xc7\xcd\xa3\xb0\xe5\xbc\xdb']) #涨停板价
tick.lowerLimit = float(d['\xb5\xf8\xcd\xa3\xb0\xe5\xbc\xdb']) #跌停板价
tick.volume = int(float(d['\xb3\xd6\xb2\xd6\xc1\xbf'])) #持仓量
tick.high = float(d['\xd7\xee\xb8\xdf\xbc\xdb']) # 最高价
tick.low = float(d['\xd7\xee\xb5\xcd\xbc\xdb']) # 最低价
tick.date = d['\xbd\xbb\xd2\xd7\xc8\xd5'] #交易日
#tick.date = datetime.strptime(d['date'], '%Y%m%d').strftime('%Y%m%d')
if d['\xd7\xee\xba\xf3\xd0\xde\xb8\xc4\xba\xc1\xc3\xeb'] == '500': #最后修改毫秒
tick.time = d[
'\xd7\xee\xba\xf3\xd0\xde\xb8\xc4\xca\xb1\xbc\xe4'] + '.5' #最后修改时间
else:
tick.time = d[
'\xd7\xee\xba\xf3\xd0\xde\xb8\xc4\xca\xb1\xbc\xe4'] + '.0' #最后修改时间
tick.datetime = datetime.strptime(tick.date + ' ' + tick.time,
'%Y%m%d %H:%M:%S.%f')
#21点以前的算到前一天
if (datetime.strptime('20010101 20:59:59.0', '%Y%m%d %H:%M:%S.%f') <
datetime.strptime('20010101' + ' ' + tick.time,
'%Y%m%d %H:%M:%S.%f') < datetime.strptime(
'20010101 23:30:01.0',
'%Y%m%d %H:%M:%S.%f')):
tick.datetime = tick.datetime + timedelta(days=-1)
flt = {'datetime': tick.datetime}
collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
elif (datetime.strptime('20010101 8:59:59.0', '%Y%m%d %H:%M:%S.%f') <
datetime.strptime('20010101' + ' ' + tick.time,
'%Y%m%d %H:%M:%S.%f') <
datetime.strptime('20010101 15:00:01.0', '%Y%m%d %H:%M:%S.%f')):
flt = {'datetime': tick.datetime}
collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
else:
#把不在开盘时间内的去掉
print u'抛弃数据：%s ' % tick.time
csv_file.close()
print u'插入完毕，耗时：%s' % (time() - start)
def __init__(self):
"""Constructor"""
host, port = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()