def __init__(self):
"""Constructor"""
host, port, logging = loadMongoSetting()
print(host)
self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()
def __init__(self):
"""Constructor"""
# MongoDB数据库相关
self.dbClient = None # MongoDB客户端对象
# host, port = loadMongoSetting()
#
# self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()
self.dbConnect()
class HistoryDataEngine(object):
"""CTA模块用的历史数据引擎"""
#----------------------------------------------------------------------
def __init__(self):
"""Constructor"""
host, port = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()
#----------------------------------------------------------------------
def lastTradeDate(self):
"""获取最近交易日（只考虑工作日，无法检查国内假期）"""
today = datetime.now()
oneday = timedelta(1)
if today.weekday() == 5:
today = today - oneday
elif today.weekday() == 6:
today = today - oneday * 2
return today.strftime('%Y%m%d')
#----------------------------------------------------------------------
def readFuturesProductSymbol(self):
"""查询所有期货产品代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return set([d['productSymbol'] for d in cx]) # 这里返回的是集合（因为会重复）
#----------------------------------------------------------------------
def readFuturesSymbol(self):
"""查询所有期货合约代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return [d['symbol'] for d in cx] # 这里返回的是列表
#----------------------------------------------------------------------
def downloadFuturesSymbol(self, tradeDate=''):
"""下载所有期货的代码"""
if not tradeDate:
tradeDate = self.lastTradeDate()
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].ensure_index(
[('symbol', pymongo.ASCENDING)], unique=True)
path = 'api/market/getMktMFutd.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
symbolDict['productSymbol'] = d['contractObject']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].update_one(
flt, {'$set': symbolDict}, upsert=True)
print u'期货合约代码下载完成'
else:
print u'期货合约代码下载失败'
#----------------------------------------------------------------------
def downloadFuturesDailyBar(self, symbol):
"""
下载期货合约的日行情，symbol是合约代码，
若最后四位为0000（如IF0000），代表下载连续合约。
"""
print u'开始下载%s日行情' % symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 主力合约
if '0000' in symbol:
path = 'api/market/getMktMFutd.json'
params = {}
params['contractObject'] = symbol.replace('0000', '')
params['mainCon'] = 1
if last:
params['startDate'] = last['date']
# 交易合约
else:
path = 'api/market/getMktFutd.json'
params = {}
params['ticker'] = symbol
if last:
params['startDate'] = last['date']
# 开始下载数据
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
bar.openInterest = d.get('openInt', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
#----------------------------------------------------------------------
def downloadAllFuturesDailyBar(self):
"""下载所有期货的主力合约日行情"""
start = time()
print u'开始下载所有期货的主力合约日行情'
productSymbolSet = self.readFuturesProductSymbol()
print u'代码列表读取成功，产品代码：%s' % productSymbolSet
# 这里也测试了线程池，但可能由于下载函数中涉及较多的数据格
# 式转换，CPU开销较大，多线程效率并无显著改变。
#p = ThreadPool(10)
#p.map(self.downloadFuturesDailyBar, productSymbolSet)
#p.close()
#p.join()
for productSymbol in productSymbolSet:
self.downloadFuturesDailyBar(productSymbol + '0000')
print u'所有期货的主力合约日行情已经全部下载完成, 耗时%s秒' % (time() - start)
#----------------------------------------------------------------------
def downloadFuturesIntradayBar(self, symbol):
"""下载期货的日内分钟行情"""
print u'开始下载%s日内分钟行情' % symbol
# 日内分钟行情只有具体合约
path = 'api/market/getFutureBarRTIntraDay.json'
params = {}
params['instrumentID'] = symbol
params['unit'] = 1
data = self.datayesClient.downloadData(path, params)
if data:
today = datetime.now().strftime('%Y%m%d')
# 创建datetime索引
self.dbClient[MINUTE_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = today
bar.time = d.get('barTime', '')
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M')
bar.volume = d.get('totalVolume', 0)
bar.openInterest = 0
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[MINUTE_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
#----------------------------------------------------------------------
def downloadEquitySymbol(self, tradeDate=''):
"""下载所有股票的代码"""
if not tradeDate:
tradeDate = self.lastTradeDate()
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].ensure_index(
[('symbol', pymongo.ASCENDING)], unique=True)
path = 'api/market/getMktEqud.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].update_one(
flt, {'$set': symbolDict}, upsert=True)
print u'股票代码下载完成'
else:
print u'股票代码下载失败'
#----------------------------------------------------------------------
def downloadEquityDailyBar(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print u'开始下载%s日行情' % symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
path = 'api/market/getMktEqud.json'
params = {}
params['ticker'] = symbol
if last:
params['beginDate'] = last['date']
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
def __init__(self):
"""Constructor"""
self.dbClient = pymongo.MongoClient()
self.datayesClient = DatayesClient()
def __init__(self):
"""Constructor"""
host, port = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()
class HistoryDataEngine(object):
"""CTA模块用的历史数据引擎"""
#----------------------------------------------------------------------
def __init__(self):
"""Constructor"""
host, port = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()
#----------------------------------------------------------------------
def lastTradeDate(self):
"""获取最近交易日（只考虑工作日，无法检查国内假期）"""
today = datetime.now()
oneday = timedelta(1)
if today.weekday() == 5:
today = today - oneday
elif today.weekday() == 6:
today = today - oneday*2
return today.strftime('%Y%m%d')
#----------------------------------------------------------------------
def readFuturesProductSymbol(self):
"""查询所有期货产品代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return set([d['productSymbol'] for d in cx]) # 这里返回的是集合（因为会重复）
#----------------------------------------------------------------------
def readFuturesSymbol(self):
"""查询所有期货合约代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return [d['symbol'] for d in cx] # 这里返回的是列表
#----------------------------------------------------------------------
def downloadFuturesSymbol(self, tradeDate=''):
"""下载所有期货的代码"""
if not tradeDate:
tradeDate = self.lastTradeDate()
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].ensure_index([('symbol', pymongo.ASCENDING)],
unique=True)
path = 'api/market/getMktMFutd.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
symbolDict['productSymbol'] = d['contractObject']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].update_one(flt, {'$set':symbolDict},
upsert=True)
print u'期货合约代码下载完成'
else:
print u'期货合约代码下载失败'
#----------------------------------------------------------------------
def downloadFuturesDailyBar(self, symbol):
"""
下载期货合约的日行情，symbol是合约代码，
若最后四位为0000（如IF0000），代表下载连续合约。
"""
print u'开始下载%s日行情' %symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 主力合约
if '0000' in symbol:
path = 'api/market/getMktMFutd.json'
params = {}
params['contractObject'] = symbol.replace('0000', '')
params['mainCon'] = 1
if last:
params['startDate'] = last['date']
# 交易合约
else:
path = 'api/market/getMktFutd.json'
params = {}
params['ticker'] = symbol
if last:
params['startDate'] = last['date']
# 开始下载数据
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
bar.openInterest = d.get('openInt', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
#----------------------------------------------------------------------
def downloadAllFuturesDailyBar(self):
"""下载所有期货的主力合约日行情"""
start = time()
print u'开始下载所有期货的主力合约日行情'
productSymbolSet = self.readFuturesProductSymbol()
print u'代码列表读取成功，产品代码：%s' %productSymbolSet
# 这里也测试了线程池，但可能由于下载函数中涉及较多的数据格
# 式转换，CPU开销较大，多线程效率并无显著改变。
#p = ThreadPool(10)
#p.map(self.downloadFuturesDailyBar, productSymbolSet)
#p.close()
#p.join()
for productSymbol in productSymbolSet:
self.downloadFuturesDailyBar(productSymbol+'0000')
print u'所有期货的主力合约日行情已经全部下载完成, 耗时%s秒' %(time()-start)
#----------------------------------------------------------------------
def downloadFuturesIntradayBar(self, symbol):
"""下载期货的日内分钟行情"""
print u'开始下载%s日内分钟行情' %symbol
# 日内分钟行情只有具体合约
path = 'api/market/getFutureBarRTIntraDay.json'
params = {}
params['instrumentID'] = symbol
params['unit'] = 1
data = self.datayesClient.downloadData(path, params)
if data:
today = datetime.now().strftime('%Y%m%d')
# 创建datetime索引
self.dbClient[MINUTE_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = today
bar.time = d.get('barTime', '')
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M')
bar.volume = d.get('totalVolume', 0)
bar.openInterest = 0
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[MINUTE_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
#----------------------------------------------------------------------
def downloadEquitySymbol(self, tradeDate=''):
"""下载所有股票的代码"""
if not tradeDate:
tradeDate = self.lastTradeDate()
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].ensure_index([('symbol', pymongo.ASCENDING)],
unique=True)
path = 'api/market/getMktEqud.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].update_one(flt, {'$set':symbolDict},
upsert=True)
print u'股票代码下载完成'
else:
print u'股票代码下载失败'
#----------------------------------------------------------------------
def downloadEquityDailyBar(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print u'开始下载%s日行情' %symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
path = 'api/market/getMktEqud.json'
params = {}
params['ticker'] = symbol
if last:
params['beginDate'] = last['date']
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
class HistoryDataEngine(object):
"""CTA模块用的历史数据引擎"""
#----------------------------------------------------------------------
def __init__(self):
"""Constructor"""
# MongoDB数据库相关
self.dbClient = None # MongoDB客户端对象
# host, port = loadMongoSetting()
#
# self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()
self.dbConnect()
#----------------------------------------------------------------------
def dbConnect(self):
"""连接MongoDB数据库"""
# 读取数据库配置的方法，已转移至vtFunction
if not self.dbClient:
# 读取MongoDB的设置
settingFileName = "VT_setting.json"
settingFileName = os.path.dirname(os.getcwd()) + "/" + settingFileName
host, port, replicaset, readPreference, database, userID, password = loadMongoSetting(settingFileName)
try:
# self.dbClient = pymongo.MongoClient(host+':'+str(port), replicaset=replicaset,readPreference=readPreference)
# db = self.dbClient[database]
# db.authenticate(userID, password)
self.dbClient = pymongo.MongoClient(host, port, serverSelectionTimeoutMS=500)
print u'MongoDB连接成功'
except ConnectionFailure:
print u'MongoDB连接失败'
except ValueError:
print u'MongoDB连接配置字段错误，请检查'
#----------------------------------------------------------------------
def lastTradeDate(self):
"""获取最近交易日（只考虑工作日，无法检查国内假期）"""
today = datetime.now()
oneday = timedelta(1)
if today.weekday() == 5:
today = today - oneday
elif today.weekday() == 6:
today = today - oneday*2
return today.strftime('%Y%m%d')
#----------------------------------------------------------------------
def realLastTradeDate(self):
"""获取真实最近交易日（通联交易所交易日历）"""
today = datetime.now().strftime('%Y%m%d')
path = 'api/master/getTradeCal.json'
params = {}
params['beginDate'] = today
params['endDate'] = today
params['exchangeCD'] = 'XSHG'
data = self.datayesClient.downloadData(path, params)
# 通联数据18:00更新,之前获取不到当日数据
time = datetime.now().strftime('%H:%M:%S')
if data[0]['isOpen'] == 1 and time >= '18:30:00':
return today
else:
lastDay = ''.join(data[0]['prevTradeDate'].split('-'))
return lastDay
#----------------------------------------------------------------------
def readFuturesProductSymbol(self):
"""查询所有期货产品代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return set([d['productSymbol'] for d in cx]) # 这里返回的是集合（因为会重复）
#----------------------------------------------------------------------
def readFuturesSymbol(self):
"""查询所有期货合约代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return [d['symbol'] for d in cx] # 这里返回的是列表
#----------------------------------------------------------------------
def downloadFuturesSymbol(self, tradeDate=''):
"""下载所有期货的代码"""
if not tradeDate:
tradeDate = self.reallastTradeDate()
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].ensure_index([('symbol', pymongo.ASCENDING)],
unique=True)
path = 'api/market/getMktMFutd.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
symbolDict['productSymbol'] = d['contractObject']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].update_one(flt, {'$set':symbolDict},
upsert=True)
print u'期货合约代码下载完成'
else:
print u'期货合约代码下载失败'
#----------------------------------------------------------------------
def downloadFuturesDailyBar(self, symbol):
"""
下载期货合约的日行情，symbol是合约代码，
若最后四位为0000（如IF0000），代表下载连续合约。
"""
print u'开始下载%s日行情' %symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 主力合约
if '0000' in symbol:
path = 'api/market/getMktMFutd.json'
params = {}
params['contractObject'] = symbol.replace('0000', '')
params['mainCon'] = 1
if last:
params['startDate'] = last['date']
# 交易合约
else:
path = 'api/market/getMktFutd.json'
params = {}
params['ticker'] = symbol
if last:
params['startDate'] = last['date']
# 开始下载数据
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
bar.openInterest = d.get('openInt', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
#----------------------------------------------------------------------
def downloadAllFuturesDailyBar(self):
"""下载所有期货的主力合约日行情"""
start = time()
print u'开始下载所有期货的主力合约日行情'
productSymbolSet = self.readFuturesProductSymbol()
print u'代码列表读取成功，产品代码：%s' %productSymbolSet
# 这里也测试了线程池，但可能由于下载函数中涉及较多的数据格
# 式转换，CPU开销较大，多线程效率并无显著改变。
#p = ThreadPool(10)
#p.map(self.downloadFuturesDailyBar, productSymbolSet)
#p.close()
#p.join()
for productSymbol in productSymbolSet:
self.downloadFuturesDailyBar(productSymbol+'0000')
print u'所有期货的主力合约日行情已经全部下载完成, 耗时%s秒' %(time()-start)
#----------------------------------------------------------------------
def downloadFuturesIntradayBar(self, symbol):
"""下载期货的日内分钟行情"""
print u'开始下载%s日内分钟行情' %symbol
# 日内分钟行情只有具体合约
path = 'api/market/getFutureBarRTIntraDay.json'
params = {}
params['instrumentID'] = symbol
params['unit'] = 1
data = self.datayesClient.downloadData(path, params)
if data:
today = datetime.now().strftime('%Y%m%d')
# 创建datetime索引
self.dbClient[MINUTE_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = today
bar.time = d.get('barTime', '')
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M')
bar.volume = d.get('totalVolume', 0)
bar.openInterest = 0
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[MINUTE_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
#----------------------------------------------------------------------
def downloadFutureIntraDayTick(self, symbol):
"""下载期货的日内tick行情"""
print u'开始下载%s日内tick行情' %symbol
# 日内分钟行情只有具体合约
st = ts.Market()
data = st.FutureTickRTIntraDay(instrumentID=symbol)
filename = 'D:\\' + symbol + '.csv'
data.to_csv(filename)
# if data:
# today = datetime.now().strftime('%Y%m%d')
#
# # 创建datetime索引
# self.dbClient[MINUTE_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
# unique=True)
#
# for d in data:
# bar = CtaBarData()
# bar.vtSymbol = symbol
# bar.symbol = symbol
# try:
# bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
# bar.open = d.get('openPrice', 0)
# bar.high = d.get('highestPrice', 0)
# bar.low = d.get('lowestPrice', 0)
# bar.close = d.get('closePrice', 0)
# bar.date = today
# bar.time = d.get('barTime', '')
# bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M')
# bar.volume = d.get('totalVolume', 0)
# bar.openInterest = 0
# except KeyError:
# print d
#
# flt = {'datetime': bar.datetime}
# self.dbClient[MINUTE_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
#
# print u'%s下载完成' %symbol
# else:
# print u'找不到合约%s' %symbol
#----------------------------------------------------------------------
#----------------------------------------------------------------------
def downloadEquitySymbol(self, tradeDate=''):
"""下载所有股票的代码"""
if not tradeDate:
tradeDate = self.lastTradeDate()
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].ensure_index([('symbol', pymongo.ASCENDING)],
unique=True)
path = 'api/market/getMktEqud.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].update_one(flt, {'$set':symbolDict},
upsert=True)
print u'股票代码下载完成'
else:
print u'股票代码下载失败'
#----------------------------------------------------------------------
def downloadEquityDailyBar(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print u'开始下载%s日行情' %symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
path = 'api/market/getMktEqud.json'
params = {}
params['ticker'] = symbol
if last:
params['beginDate'] = last['date']
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index([('datetime', pymongo.ASCENDING)],
unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(flt, {'$set':bar.__dict__}, upsert=True)
print u'%s下载完成' %symbol
else:
print u'找不到合约%s' %symbol
def __init__(self):
"""Constructor"""
self.dbClient = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
self.datayesClient = DatayesClient()
def __init__(self):
"""Constructor"""
self.dbClient = pymongo.MongoClient()
self.datayesClient = DatayesClient()
class HistoryDataEngine(object):
"""CTA模块用的历史数据引擎"""
#----------------------------------------------------------------------
def __init__(self):
"""Constructor"""
host, port, logging = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()
#----------------------------------------------------------------------
def lastTradeDate(self):
"""获取最近交易日（只考虑工作日，无法检查国内假期）"""
'''2017052500 Add by hetajen begin'''
today = datetime.datetime.now()
oneday = datetime.timedelta(1)
'''2017052500 Add by hetajen end'''
if today.weekday() == 5:
today = today - oneday
elif today.weekday() == 6:
today = today - oneday * 2
return today.strftime('%Y%m%d')
# ----------------------------------------------------------------------
'''2017052500 Add by hetajen begin'''
def loadTradeCal(self):
print u'Trade Calendar load from csv, BEGIN'
'''数据库'''
dbName = SETTING_DB_NAME
collectionName = TRADECAL_CL_NAME
collection = self.dbClient[dbName][collectionName]
collection.ensure_index([('calendarDate', pymongo.ASCENDING)],
unique=True) # 创建索引
# 查询数据库中已有数据的最后日期
cx = collection.find(sort=[('calendarDate', pymongo.DESCENDING)])
if cx.count():
last = cx[0][u'calendarDate']
else:
last = datetime.datetime(1990, 12, 31, 23, 59, 59)
'''数据源'''
import csv
fileName = os.path.join(os.path.dirname(os.path.abspath(__file__)),
u'strategy\\tradeCal.csv')
reader = csv.reader(file(fileName, 'r'))
'''数据解析&持久化'''
calendarsTmp = []
calendars = []
filterHeaders = True # 过滤表头
nextTradeDate = None
for d in reader:
if filterHeaders:
filterHeaders = False
continue
calendarDict = {}
calendarDict['calendarDate'] = datetime.datetime.strptime(
d[1].replace('/', ''), '%Y%m%d')
if calendarDict['calendarDate'] < last:
continue
calendarDict['exchangeCD'] = d[0]
calendarDict['isOpen'] = d[2]
if (d[3] != ''):
calendarDict['prevTradeDate'] = datetime.datetime.strptime(
d[3].replace('/', ''), '%Y%m%d')
else:
calendarDict['prevTradeDate'] = None
calendarDict['isWeekEnd'] = d[4]
calendarDict['isMonthEnd'] = d[5]
calendarDict['isQuarterEnd'] = d[6]
calendarDict['isYearEnd'] = d[7]
calendarsTmp.append(calendarDict)
calendarsTmp.reverse()
for calendarDict in calendarsTmp:
calendarDict['nextTradeDate'] = nextTradeDate
calendars.append(calendarDict)
if calendarDict['isOpen'] == '1':
nextTradeDate = calendarDict['calendarDate']
# calendars.reverse()
for calendarDict in calendars:
flt = {'calendarDate': calendarDict['calendarDate']}
collection.update_one(flt, {'$set': calendarDict}, upsert=True)
print u'Trade Calendar load from csv, End'
def downloadTradeCal(self):
print u'Trade Calendar download from datayes, BEGIN'
'''数据库'''
dbName = SETTING_DB_NAME
collectionName = TRADECAL_CL_NAME
collection = self.dbClient[dbName][collectionName]
collection.ensure_index([('calendarDate', pymongo.ASCENDING)],
unique=True) # 创建索引
# 查询数据库中已有数据的最后日期
cx = collection.find(sort=[('calendarDate', pymongo.DESCENDING)])
if cx.count():
last = cx[0][u'calendarDate']
else:
last = datetime.datetime(1990, 12, 31, 23, 59, 59)
'''数据源'''
path = 'api/master/getTradeCal.json'
params = {}
# params['field'] = ''
params['exchangeCD'] = 'XSGE'
params['beginDate'] = unicode(last.date())
data = self.datayesClient.downloadData(path, params)
'''数据解析&持久化'''
if data:
for d in data:
calendarDict = {}
calendarDict['exchangeCD'] = d['exchangeCD']
calendarDict['calendarDate'] = d['calendarDate']
calendarDict['isOpen'] = d['isOpen']
calendarDict['prevTradeDate'] = d['prevTradeDate']
calendarDict['isWeekEnd'] = d['isWeekEnd']
calendarDict['isMonthEnd'] = d['isMonthEnd']
calendarDict['isQuarterEnd'] = d['isQuarterEnd']
calendarDict['isYearEnd'] = d['isYearEnd']
flt = {'calendarDate': d['calendarDate']}
collection.update_one(flt, {'$set': calendarDict}, upsert=True)
print u'Trade Calendar download from datayes, END'
else:
print u'Trade Calendar download from datayes, Wrong'
'''2017052500 Add by hetajen end'''
#----------------------------------------------------------------------
def readFuturesProductSymbol(self):
"""查询所有期货产品代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return set([d['productSymbol'] for d in cx]) # 这里返回的是集合（因为会重复）
#----------------------------------------------------------------------
def readFuturesSymbol(self):
"""查询所有期货合约代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return [d['symbol'] for d in cx] # 这里返回的是列表
#----------------------------------------------------------------------
def downloadFuturesSymbol(self, tradeDate=''):
"""下载所有期货的代码"""
if not tradeDate:
tradeDate = self.lastTradeDate()
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].ensure_index(
[('symbol', pymongo.ASCENDING)], unique=True)
path = 'api/market/getMktMFutd.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
symbolDict['productSymbol'] = d['contractObject']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].update_one(
flt, {'$set': symbolDict}, upsert=True)
print u'期货合约代码下载完成'
else:
print u'期货合约代码下载失败'
#----------------------------------------------------------------------
def downloadFuturesDailyBar(self, symbol):
"""
下载期货合约的日行情，symbol是合约代码，
若最后四位为0000（如IF0000），代表下载连续合约。
"""
print u'开始下载%s日行情' % symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 主力合约
if '0000' in symbol:
path = 'api/market/getMktMFutd.json'
params = {}
params['contractObject'] = symbol.replace('0000', '')
params['mainCon'] = 1
if last:
params['startDate'] = last['date']
# 交易合约
else:
path = 'api/market/getMktFutd.json'
params = {}
params['ticker'] = symbol
if last:
params['startDate'] = last['date']
# 开始下载数据
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
'''2017052500 Add by hetajen begin'''
bar.datetime = datetime.datetime.strptime(
bar.date, '%Y%m%d')
'''2017052500 Add by hetajen end'''
bar.volume = d.get('turnoverVol', 0)
bar.openInterest = d.get('openInt', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
#----------------------------------------------------------------------
def downloadAllFuturesDailyBar(self):
"""下载所有期货的主力合约日行情"""
'''2017052500 Add by hetajen begin'''
start = time.time()
'''2017052500 Add by hetajen end'''
print u'开始下载所有期货的主力合约日行情'
productSymbolSet = self.readFuturesProductSymbol()
print u'代码列表读取成功，产品代码：%s' % productSymbolSet
# 这里也测试了线程池，但可能由于下载函数中涉及较多的数据格
# 式转换，CPU开销较大，多线程效率并无显著改变。
#p = ThreadPool(10)
#p.map(self.downloadFuturesDailyBar, productSymbolSet)
#p.close()
#p.join()
for productSymbol in productSymbolSet:
self.downloadFuturesDailyBar(productSymbol + '0000')
'''2017052500 Add by hetajen begin'''
print u'所有期货的主力合约日行情已经全部下载完成, 耗时%s秒' % (time.time() - start)
'''2017052500 Add by hetajen end'''
#----------------------------------------------------------------------
def downloadFuturesIntradayBar(self, symbol):
"""下载期货的日内分钟行情"""
print u'开始下载%s日内分钟行情' % symbol
# 日内分钟行情只有具体合约
path = 'api/market/getFutureBarRTIntraDay.json'
params = {}
params['instrumentID'] = symbol
params['unit'] = 1
data = self.datayesClient.downloadData(path, params)
if data:
'''2017052500 Add by hetajen begin'''
today = datetime.datetime.now().strftime('%Y%m%d')
'''2017052500 Add by hetajen end'''
# 创建datetime索引
self.dbClient[MINUTE_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = today
bar.time = d.get('barTime', '')
'''2017052500 Add by hetajen begin'''
bar.datetime = datetime.datetime.strptime(
bar.date + ' ' + bar.time, '%Y%m%d %H:%M')
'''2017052500 Add by hetajen end'''
bar.volume = d.get('totalVolume', 0)
bar.openInterest = 0
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[MINUTE_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
#----------------------------------------------------------------------
def downloadEquitySymbol(self, tradeDate=''):
"""下载所有股票的代码"""
if not tradeDate:
tradeDate = self.lastTradeDate()
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].ensure_index(
[('symbol', pymongo.ASCENDING)], unique=True)
path = 'api/market/getMktEqud.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].update_one(
flt, {'$set': symbolDict}, upsert=True)
print u'股票代码下载完成'
else:
print u'股票代码下载失败'
#----------------------------------------------------------------------
def downloadEquityDailyBar(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print u'开始下载%s日行情' % symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
path = 'api/market/getMktEqud.json'
params = {}
params['ticker'] = symbol
if last:
params['beginDate'] = last['date']
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
'''2017052500 Add by hetajen begin'''
bar.datetime = datetime.datetime.strptime(
bar.date, '%Y%m%d')
'''2017052500 Add by hetajen end'''
bar.volume = d.get('turnoverVol', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
class HistoryDataEngine(object):
"""CTA模块用的历史数据引擎"""
# ----------------------------------------------------------------------
def __init__(self):
"""Constructor"""
host, port = loadMongoSetting()
self.dbClient = pymongo.MongoClient(host, port)
self.datayesClient = DatayesClient()
# ----------------------------------------------------------------------
def lastTradeDate(self):
"""获取最近交易日（只考虑工作日，无法检查国内假期）"""
today = datetime.now()
oneday = timedelta(1)
if today.weekday() == 5:
today = today - oneday
elif today.weekday() == 6:
today = today - oneday * 2
return today.strftime('%Y%m%d')
# ----------------------------------------------------------------------
def readFuturesProductSymbol(self):
"""查询所有期货产品代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return set([d['productSymbol'] for d in cx]) # 这里返回的是集合（因为会重复）
# ----------------------------------------------------------------------
def readFuturesSymbol(self):
"""查询所有期货合约代码"""
cx = self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].find()
return [d['symbol'] for d in cx] # 这里返回的是列表
# ----------------------------------------------------------------------
def downloadFuturesSymbol(self, tradeDate=''):
"""下载所有期货的代码"""
if not tradeDate:
tradeDate = self.lastTradeDate()
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].ensure_index(
[('symbol', pymongo.ASCENDING)], unique=True)
path = 'api/market/getMktMFutd.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
symbolDict['productSymbol'] = d['contractObject']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['FuturesSymbol'].update_one(
flt, {'$set': symbolDict}, upsert=True)
print u'期货合约代码下载完成'
else:
print u'期货合约代码下载失败'
# ----------------------------------------------------------------------
def downloadFuturesBar(self, symbol, unit):
"""
下载期货合约的日、周、月行情，symbol是合约代码，unit包括daily、weekly、monthly，
"""
print u'开始下载%s %s行情' % (symbol, unit)
# unit包括daily、weekly、monthly，根据unit值对DB_NAME和path进行赋值
DB_NAME = ""
if unit == "daily":
DB_NAME = DAILY_DB_NAME
path = 'api/market/getMktFutd.json'
elif unit == "weekly":
DB_NAME = WEEKLY_DB_NAME
path = 'api/market/getMktFutw.json'
elif unit == "monthly":
DB_NAME = MONTHLY_DB_NAME
path = 'api/market/getMktFutm.json'
else:
print "参数错误"
return
# # 查询数据库中已有数据的最后日期
# cl = self.dbClient[DB_NAME][symbol]
# cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
# if cx.count():
# last = cx[0]
# else:
# last = ''
# # 主力合约
# if '0000' in symbol:
# path = 'api/market/getMktMFutd.json'
#
# params = {}
# params['contractObject'] = symbol.replace('0000', '')
# params['mainCon'] = 1
# if last:
# params['startDate'] = last['date']
# 交易合约
params = {}
params['ticker'] = symbol
# if last:
# params['beginDate'] = last['date']
# 开始下载数据
data = self.datayesClient.downloadData(path, params)
if data:
print "返回 %d 条数据记录" % len(data)
# 创建datetime索引
self.dbClient[DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
if unit == "daily":
bar.date = d.get('tradeDate', '').replace('-', '')
else:
bar.date = d.get('endDate', '').replace('-', '')
bar.tradedays = d.get('tradeDays', 0)
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
bar.openInterest = d.get('openInt', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'下载完成'
return data
else:
print u'找不到合约%s' % symbol
# ----------------------------------------------------------------------
# def downloadAllFuturesDailyBar(self):
# """下载所有期货的主力合约日行情"""
# start = time()
# print u'开始下载所有期货的主力合约日行情'
#
# productSymbolSet = self.readFuturesProductSymbol()
#
# print u'代码列表读取成功，产品代码：%s' % productSymbolSet
#
# # 这里也测试了线程池，但可能由于下载函数中涉及较多的数据格
# # 式转换，CPU开销较大，多线程效率并无显著改变。
# # p = ThreadPool(10)
# # p.map(self.downloadFuturesDailyBar, productSymbolSet)
# # p.close()
# # p.join()
#
# for productSymbol in productSymbolSet:
# self.downloadFuturesDailyBar(productSymbol + '0000')
#
# print u'所有期货的主力合约日行情已经全部下载完成, 耗时%s秒' % (time() - start)
#
# ----------------------------------------------------------------------
def downloadFuturesIntradayBar(self, symbol, unit):
"""下载期货的日内分钟行情"""
print u'开始下载%s日内分钟行情' % symbol
# 日内分钟行情只有具体合约
path = 'api/market/getFutureBarRTIntraDay.json'
params = {}
params['instrumentID'] = symbol
params['unit'] = unit
data = self.datayesClient.downloadData(path, params)
# print u"datayesClient返回值: ", data
if data:
print "返回 %d 条数据记录" % len(data)
today = datetime.now().strftime('%Y%m%d')
yesterday = (datetime.now() - timedelta(1)).strftime('%Y%m%d')
#给数据库命名
dbname = ''
if unit == 1:
dbname = MINUTE_DB_NAME
elif unit == 5:
dbname = MINUTE5_DB_NAME
elif unit == 15:
dbname = MINUTE15_DB_NAME
elif unit == 30:
dbname = MINUTE30_DB_NAME
elif unit == 60:
dbname = MINUTE60_DB_NAME
else:
print u'分钟值错误'
# 创建datetime索引
self.dbClient[dbname][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
if d.get('openPrice', 0) == d.get('highPrice', 0) == d.get(
'lowPrice', 0) == d.get('closePrice', 0):
continue
else:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = ''
bar.open = d.get('openPrice', 0)
bar.high = d.get('highPrice', 0)
bar.low = d.get('lowPrice', 0)
bar.close = d.get('closePrice', 0)
bar.time = d.get('barTime', '')
if bar.time[0] == '2':
bar.date = yesterday
else:
bar.date = today
bar.datetime = datetime.strptime(
bar.date + ' ' + bar.time, '%Y%m%d %H:%M')
bar.volume = d.get('totalValue', 0)
bar.openInterest = d.get('openInterest', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[dbname][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'下载完成'
else:
print u'找不到合约%s' % symbol
# ----------------------------------------------------------------------
def downloadEquitySymbol(self, tradeDate=''):
"""下载所有股票的代码"""
if not tradeDate:
tradeDate = self.lastTradeDate()
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].ensure_index(
[('symbol', pymongo.ASCENDING)], unique=True)
path = 'api/market/getMktEqud.json'
params = {}
params['tradeDate'] = tradeDate
data = self.datayesClient.downloadData(path, params)
if data:
for d in data:
symbolDict = {}
symbolDict['symbol'] = d['ticker']
flt = {'symbol': d['ticker']}
self.dbClient[SETTING_DB_NAME]['EquitySymbol'].update_one(
flt, {'$set': symbolDict}, upsert=True)
print u'股票代码下载完成'
else:
print u'股票代码下载失败'
# ----------------------------------------------------------------------
def downloadEquityDailyBar(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print u'开始下载%s日行情' % symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
path = 'api/market/getMktEqud.json'
params = {}
params['ticker'] = symbol
if last:
params['beginDate'] = last['date']
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
