def loadMcCsv(fileName, dbName, symbol):
"""将Multicharts导出的csv格式的历史数据插入到Mongo数据库中"""
start = time()
print(u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
with open(fileName, 'r') as f:
reader = csv.DictReader(f)
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d['Open'])
bar.high = float(d['High'])
bar.low = float(d['Low'])
bar.close = float(d['Close'])
bar.date = datetime.strptime(d['Date'],
'%Y-%m-%d').strftime('%Y%m%d')
bar.time = d['Time']
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d['TotalVolume']
bar.lastBar = False
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print(bar.date, bar.time)
print(u'插入完毕，耗时：%s' % (time() - start))
def onQueryBar(self, data, request):
""""""
e = request.extra.split('_')
l = len(data)
index = 0
for d in reversed(data):
bar = VtBarData()
bar.symbol = d['symbol'] # 代码
bar.interval = e[1] # K线周期.
bar.exchange = EXCHANGE_BITMEX
bar.vtSymbol = '.'.join([bar.symbol, bar.exchange])
bar.open = float(d['open']) # OHLCV
bar.high = float(d['high'])
bar.low = float(d['low'])
bar.close = float(d['close'])
bar.volume = float(d['volume'])
dt = datetime.strptime(d['timestamp'][:-1]+'000Z', '%Y-%m-%dT%H:%M:%S.%fZ')
if 'm' in bar.interval:
bar.datetime = dt - timedelta(minutes=int(bar.interval[:-1]))
elif 'd' in bar.interval:
bar.datetime = dt - timedelta(days=int(bar.interval[:-1]))
elif 'h' in bar.interval:
bar.datetime = dt - timedelta(hours=int(bar.interval[:-1]))
bar.date = bar.datetime.strftime('%Y-%m-%d')
bar.time = bar.datetime.strftime('%H:%M:%S.%f')[:-3]
index += 1
if index == l:
bar.lastBar = True
self.gateway.onBar(bar)
def loadTdxCsv(fileName, dbName, symbol):
"""将通达信导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
for d in reader:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.date = datetime.strptime(d[0], '%Y/%m/%d').strftime('%Y%m%d')
bar.time = d[1][:2] + ':' + d[1][2:4] + ':00'
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M:%S')
bar.volume = d[6]
bar.openInterest = d[7]
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print bar.date, bar.time
print u'插入完毕，耗时：%s' % (time() - start)
def downloadEquityDailyBar(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print u'开始下载%s日行情' % symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
path = 'api/market/getMktEqud.json'
params = {}
params['ticker'] = symbol
if last:
params['beginDate'] = last['date']
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
def downloadEquityDailyBarts(self, symbol):
"""
下载股票的日行情，symbol是股票代码
"""
print(u'开始下载%s日行情' % symbol)
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 开始下载数据
import tushare as ts
if last:
start = last['date'][:4] + '-' + last['date'][4:6] + '-' + last[
'date'][6:]
data = ts.get_k_data(symbol, start)
if not data.empty:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for index, d in data.iterrows():
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.open = d.get('open')
bar.high = d.get('high')
bar.low = d.get('low')
bar.close = d.get('close')
bar.date = d.get('date').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('volume')
except KeyError:
print(d)
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print(u'%s下载完成' % symbol)
else:
print(u'找不到合约%s' % symbol)
def downloadFuturesIntradayBar(self, symbol):
"""下载期货的日内分钟行情"""
print u'开始下载%s日内分钟行情' % symbol
# 日内分钟行情只有具体合约
path = 'api/market/getFutureBarRTIntraDay.json'
params = {}
params['instrumentID'] = symbol
params['unit'] = 1
data = self.datayesClient.downloadData(path, params)
if data:
today = datetime.now().strftime('%Y%m%d')
# 创建datetime索引
self.dbClient[MINUTE_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = today
bar.time = d.get('barTime', '')
bar.datetime = datetime.strptime(bar.date + ' ' + bar.time,
'%Y%m%d %H:%M')
bar.volume = d.get('totalVolume', 0)
bar.openInterest = 0
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[MINUTE_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol
def generateVtBar(row):
"""生成K线"""
bar = VtBarData()
bar.symbol = row['code']
bar.exchange = ''
bar.vtSymbol = bar.symbol
bar.open = row['open']
bar.high = row['high']
bar.low = row['low']
bar.close = row['close']
bar.volume = row['volume']
bar.datetime = datetime.strptime(row['time_key'], '%Y-%m-%d %H:%M:%S')
bar.date = bar.datetime.strftime("%Y%m%d")
bar.time = bar.datetime.strftime("%H:%M:%S")
return bar
def generateVtBar(d):
"""生成K线"""
bar = VtBarData()
bar.symbol = d['symbol']
bar.vtSymbol = d['symbol']
bar.date = d['date']
bar.time = ':'.join([d['time'][:2], d['time'][2:]])
bar.open = d['open']
bar.high = d['high']
bar.low = d['low']
bar.close = d['close']
bar.volume = d['volume']
bar.openInterest = d['openInterest']
bar.datetime = datetime.datetime.strptime(' '.join([bar.date, bar.time]),
'%Y%m%d %H:%M')
return bar
def loadBitmexCsv(fileName, symbol, dbName=None):
barlist = []
reader = csv.reader(open(fileName, "r"))
for d in reader:
if 'open' not in d[1]:
bar = VtBarData()
bar.symbol = symbol
bar.interval = '1m' # K线周期.
bar.exchange = EXCHANGE_BITMEX
bar.vtSymbol = '.'.join([bar.symbol, bar.exchange])
bar.datetime = datetime.strptime(d[0], '%Y-%m-%d %H:%M:%S')
bar.date = bar.datetime.date().strftime('%Y-%m-%d')
bar.time = bar.datetime.time().strftime('%H:%M:%S')
bar.open = float(d[1])
bar.high = float(d[2])
bar.low = float(d[3])
bar.close = float(d[4])
bar.volume = float(d[5])
barlist.append(bar)
return barlist
def generateVtBar(row):
"""生成K线"""
bar = VtBarData()
symbol, exchange = row['symbol'].split('.')
bar.symbol = symbol
bar.exchange = exchangeMapReverse[exchange]
bar.vtSymbol = '.'.join([bar.symbol, bar.exchange])
bar.open = row['open']
bar.high = row['high']
bar.low = row['low']
bar.close = row['close']
bar.volume = row['volume']
bar.date = str(row['date'])
bar.time = str(row['time']).rjust(6, '0')
# 将bar的时间改成提前一分钟
hour = bar.time[0:2]
minute = bar.time[2:4]
sec = bar.time[4:6]
if minute == "00":
minute = "59"
h = int(hour)
if h == 0:
h = 24
hour = str(h - 1).rjust(2, '0')
else:
minute = str(int(minute) - 1).rjust(2, '0')
bar.time = hour + minute + sec
bar.datetime = dt.datetime.strptime(
' '.join([bar.date, bar.time]), '%Y%m%d %H%M%S')
return bar
def loadOKEXCsv(fileName, dbName, symbol):
"""将OKEX导出的csv格式的历史分钟数据插入到Mongo数据库中"""
start = time()
print(u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol))
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(open(fileName, "r"))
for d in reader:
if len(d[1]) > 10:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.datetime = datetime.strptime(d[1], '%Y-%m-%d %H:%M:%S')
bar.date = bar.datetime.date().strftime('%Y%m%d')
bar.time = bar.datetime.time().strftime('%H:%M:%S')
bar.open = float(d[2])
bar.high = float(d[3])
bar.low = float(d[4])
bar.close = float(d[5])
bar.volume = float(d[6])
bar.tobtcvolume = float(d[7])
flt = {'datetime': bar.datetime}
collection.update_one(flt, {'$set': bar.__dict__}, upsert=True)
print('%s \t %s' % (bar.date, bar.time))
print(u'插入完毕，耗时：%s' % (time() - start))
def downloadFuturesDailyBar(self, symbol):
"""
下载期货合约的日行情，symbol是合约代码，
若最后四位为0000（如IF0000），代表下载连续合约。
"""
print u'开始下载%s日行情' % symbol
# 查询数据库中已有数据的最后日期
cl = self.dbClient[DAILY_DB_NAME][symbol]
cx = cl.find(sort=[('datetime', pymongo.DESCENDING)])
if cx.count():
last = cx[0]
else:
last = ''
# 主力合约
if '0000' in symbol:
path = 'api/market/getMktMFutd.json'
params = {}
params['contractObject'] = symbol.replace('0000', '')
params['mainCon'] = 1
if last:
params['startDate'] = last['date']
# 交易合约
else:
path = 'api/market/getMktFutd.json'
params = {}
params['ticker'] = symbol
if last:
params['startDate'] = last['date']
# 开始下载数据
data = self.datayesClient.downloadData(path, params)
if data:
# 创建datetime索引
self.dbClient[DAILY_DB_NAME][symbol].ensure_index(
[('datetime', pymongo.ASCENDING)], unique=True)
for d in data:
bar = VtBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
try:
bar.exchange = DATAYES_TO_VT_EXCHANGE.get(
d.get('exchangeCD', ''), '')
bar.open = d.get('openPrice', 0)
bar.high = d.get('highestPrice', 0)
bar.low = d.get('lowestPrice', 0)
bar.close = d.get('closePrice', 0)
bar.date = d.get('tradeDate', '').replace('-', '')
bar.time = ''
bar.datetime = datetime.strptime(bar.date, '%Y%m%d')
bar.volume = d.get('turnoverVol', 0)
bar.openInterest = d.get('openInt', 0)
except KeyError:
print d
flt = {'datetime': bar.datetime}
self.dbClient[DAILY_DB_NAME][symbol].update_one(
flt, {'$set': bar.__dict__}, upsert=True)
print u'%s下载完成' % symbol
else:
print u'找不到合约%s' % symbol