def loadTick(self, dbName, collectionName, days):
"""从数据库中读取Tick数据，startDate是datetime对象"""
startDate = self.today - timedelta(days)
d = {'datetime':{'$gte':startDate}}
tickData = self.mainEngine.dbQuery(dbName, collectionName, d)
l = []
for d in tickData:
tick = VtTickData()
tick.__dict__ = d
l.append(tick)
return l
def loadTick(self, dbName, collectionName, days):
"""从数据库中读取Tick数据，startDate是datetime对象"""
startDate = self.today - timedelta(days)
d = {'datetime':{'$gte':startDate}}
tickData = self.mainEngine.dbQuery(dbName, collectionName, d, 'datetime')
l = []
for d in tickData:
tick = VtTickData()
tick.__dict__ = d
l.append(tick)
return l
def generateVtTick(row):
'生成tick'
tick = VtTickData()
tick.symbol = row['code']
tick.exchange = generateExchange(tick.symbol)
tick.vtSymbol = '.'.ljust([tick.symbol, tick.exchange])
tick.lastPrice = row['close']
tick.datetime = row.name
tick.highPrice = row['high']
tick.lowPrice = row['low']
def to_vnpy(self, gateway):
tick = VtTickData()
tick.symbol = self.instrument
tick.exchange = EXCHANGE_OANDA
tick.gatewayName = gateway.gatewayName
tick.vtSymbol = VN_SEPARATOR.join([tick.symbol, tick.gatewayName])
tick.datetime, tick.date, tick.time = parse_datetime_str(self.time)
ibids = list(range(len(self.bids)))
iasks = list(range(len(self.asks)))
bids = {
"bidPrice%s" % (i + 1): float(v["price"])
for i, v in zip(ibids, self.bids)
}
bid_volumes = {
"bidVolume%s" % (i + 1): v["liquidity"]
for i, v in zip(ibids, self.bids)
}
asks = {
"askPrice%s" % (i + 1): float(v["price"])
for i, v in zip(iasks, self.asks)
}
ask_volumes = {
"askVolume%s" % (i + 1): v['liquidity']
for i, v in zip(iasks, self.asks)
}
tick.__dict__.update(bids)
tick.__dict__.update(bid_volumes)
tick.__dict__.update(asks)
tick.__dict__.update(ask_volumes)
tick.lastPrice = float(
Decimal(str((tick.askPrice1 + tick.bidPrice1) / 2.0)).quantize(
Decimal(str(tick.askPrice1))))
return {
VtTickData: [tick],
}
def loadTick(self, dbName, collectionName, days):
"""从数据库中读取Tick数据，startDate是datetime对象"""
print "%s.%s.%s" % (__name__, self.__class__.__name__, get_current_function_name())
startDate = self.today - timedelta(days)
d = {'datetime': {'$gte': startDate}}
tickData = self.mainEngine.dbQuery(dbName, collectionName, d, 'datetime')
l = []
for d in tickData:
tick = VtTickData()
tick.__dict__ = d
l.append(tick)
return l
def startWork(self):
self.bm = BarGenerator(self.onBar, self.kLine, self.onXminBar)
currentDate = None
newData = False
startTime = 0
count = 0
startDate = self.startDateFromHistory(self.symbol)
if not startDate:
tickCursor = self.tickCollection.find().sort('datetime')
else:
flt = {'datetime': {'$gte': startDate}}
tickCursor = self.tickCollection.find(flt).sort('datetime')
for d in tickCursor:
tick = VtTickData()
tick.__dict__ = d
tickDateStr = tick.datetime.strftime('%Y%m%d')
tickDate = datetime.strptime(tickDateStr, '%Y%m%d')
if not currentDate or currentDate != tickDate:
# 新的一天k线初始化
self.newDayTick = tick
self.bm.manualInit()
self.newDayTick = None
if newData:
# 保存导入记录
self.updateHistory(self.symbol, currentDate)
sub = time() - startTime
print u'用时：', sub, 's'
print u'数据量：', count, '\n'
count = 0
currentDate = tickDate
# 检查是否有数据库记录
if not self.needUpdateChecking(self.symbol, currentDate):
# 当天的数据已经存在，无需重复导入
newData = False
else:
print '*' * 6, tick.datetime.date(), '*' * 6
newData = True
startTime = time()
if newData:
self.bm.updateTick(tick)
count += 1
print u'所有数据导入完成'
def get_current_tick(symbol,type_=ProductClass.Future):
"""查询合约当前最新的价格"""
redis = instance.datasourceManager.get('redis').conn
key_name = ''
try:
if type_ == ProductClass.Future:
key_name = CtpMarketSymbolTickFormat.format(symbol=symbol)
elif type_ == ProductClass.Stock:
key_name = XtpMarketSymbolTickFormat.format(symbol=symbol)
data = redis.hgetall(key_name)
tick = VtTickData()
tick.__dict__ = data
except:
return None
return tick
def get_symbol_ticks(message, ctx):
"""订阅的所有合约行情数据"""
topic = ctx.get('name')
data = message
# topic: 订阅的通道名称 ctp_ticks_*
symbol = topic.split('_')[-1]
tick = json.loads(data)
tick['datetime'] = datetime.datetime.strptime(
' '.join([tick.get('date'), tick.get('time')]), '%Y%m%d %H:%M:%S.%f')
tickobj = VtTickData()
tickobj.__dict__ = tick
main = instance.serviceManager.get('main')
# debug = main.cfgs.get('debug',{})
# if debug.get('enable',False) and symbol in debug.get('symbols',[]):
# main.onTick(symbol,tickobj)
# else:
main.onTick(symbol, tickobj)
"""
def __init__(self, arbEngine, setting):
"""Constructor"""
super(ArbStrategy, self).__init__(arbEngine, setting)
# 初始化所有变量
self.tick1 = VtTickData() # 合约1 Tick
self.tick2 = VtTickData() # 合约2 Tick
self.tick3 = VtTickData() # 合约3 Tick
self.tick4 = VtTickData() # 合约4 Tick
self.price1 = EMPTY_FLOAT # 合约1 成交价
self.price2 = EMPTY_FLOAT # 合约2 成交价
self.price3 = EMPTY_FLOAT # 合约3 成交价
self.price4 = EMPTY_FLOAT # 合约4 成交价
self.varDirection = u'' # 方向
self.varOffset = u'' # 开平
self.askPrice1 = EMPTY_FLOAT # 当前对价可买价
self.bidPrice1 = EMPTY_FLOAT # 当前对价可卖价
self.contractF = u'' # 组合公式表示
self.ncon = 0 # 操作合约数
self.symbolList = [] # 所有操作合约
self.aPos = 0 # 总持仓数
self.aCost = 0 # 平均持仓成本
self.lastVolume = 0 # 上次发单数
self.volumeRemain = 0 # 剩余订单
self.lastVolumeC1 = 0 # C1 上次发单数
self.Lock = False # 发单操作锁,防止重复发单
self.refreshCost = False # 刷新成本,防止重复
self.tickState = TICK_STOP # 状态机状态
self.productClass = PRODUCT_FOREX # 产品类型（只有IB接口需要）
self.currency = EMPTY_STRING # 货币（只有IB接口需要）
def make_min_bar_real_calc(symbol,
nmin,
kline_time,
start,
end,
drop=True,
product=ProductClass.Future):
"""
"""
# if nmin not in TimeDuration.SCALES.keys():
# print 'Error: paramter nmin:{} invalid.'.format(nmin)
# return
# mins = TimeDuration.SCALES[nmin]/TimeDuration.MINUTE
# t1 = t.replace(second=0,microsecond=0)
# t2 = t1 + TimeDelta(minutes=mins)
conn = mongodb_conn
dbname = CTP_TICK_DB
coll = conn[dbname][symbol]
# p1 = t1 - TimeDelta(minutes=leftMargin)
# p2 = t2 + TimeDelta(minutes=rightMargin)
t1 = kline_time
p1 = start
p2 = end
rs = coll.find({
'datetime': {
'$gte': p1,
'$lt': p2
},
'flag': 0
}).sort((['datetime', pymongo.ASCENDING], ['seq', pymongo.ASCENDING]))
print symbol, p1, '-->', p2 #, ' count:',rs.count()
dbname = CTP_BAR_DB
collbar_1m = conn[dbname.format(nmin)][symbol]
bar = None
# return
# 删除已存在 1m bar
if drop:
collbar_1m.remove({'datetime': t1})
if rs.count() == 0: # 分钟内无tick数据 ，将连续之前的bar
# if not prev_bar: #查找前一个bar
prev_bar = None
r = collbar_1m.find({
'datetime': {
'$lt': t1
},
'flag': 0
}).sort('datetime', pymongo.DESCENDING).limit(1)
if r.count():
prev_bar = VtBarData()
prev_bar.__dict__ = r[0]
if prev_bar:
prev_bar.datetime = t1
prev_bar.date = prev_bar.datetime.strftime('%Y%m%d')
prev_bar.time = prev_bar.datetime.strftime('%H:%M:%S.%f')
prev_bar.high = prev_bar.close
prev_bar.low = prev_bar.close
prev_bar.open = prev_bar.close
prev_bar.volume = 0
bar = prev_bar
else: # 计算当前分钟内的bar
bar = VtBarData()
tick = VtTickData()
tick.__dict__ = rs[0]
bar.datetime = t1
bar.date = bar.datetime.strftime('%Y%m%d')
bar.time = bar.datetime.strftime('%H:%M:%S.%f')
bar.vtSymbol = tick.vtSymbol
bar.symbol = tick.symbol
bar.exchange = tick.exchange
bar.open = tick.lastPrice
bar.high = tick.lastPrice
bar.low = tick.lastPrice
last = None
#找到前一个k线最后一个tick作为last
rs = list(rs)
# first = rs[0]
#first = copy.deepcopy(rs[0])
#if t.time() == Time(14,59) and nmin=='1m':
# print 'pause'
#first_time = first['datetime'].time().replace(second=0,microsecond=0)
if t1 == Time(21, 0):
bar.volume = 0
#if first_time in (Time(20,59),Time(21,0)):
# first['volume'] = 0
# last = VtTickData()
# last.__dict__ = first
# last = first
# if t.time()== Time(21,0):
# pass
#else:
_rs = coll.find({
'datetime': {
'$lt': p1
},
'flag': 0
}).sort((['datetime',
pymongo.DESCENDING], ['seq', pymongo.DESCENDING])).limit(1)
try:
last = VtTickData()
last.__dict__ = _rs[0]
except:
last = None
if last and t1 == Time(21, 0):
last.volume = 0
# if _rs.count(): # too slowly, never use..
# last = VtTickData()
# last.__dict__ = _rs[0]
# --
# ss = list(rs)
# s = ss[-1]
# print s
# return
"""
if t.time() == Time(21,9):
print t.time()
rs = list(rs)
f = open('test.txt','w')
for s in rs:
f.write('time:{} lower-price:{}\n'.format(str(t),s['lowPrice']))
f.close()
"""
for r in rs:
tick = VtTickData()
tick.__dict__ = r
bar.high = max(bar.high, tick.lastPrice)
bar.low = min(bar.low, tick.lastPrice)
bar.close = tick.lastPrice
bar.openInterest = tick.openInterest
if last:
bar.volume += (tick.volume - last.volume)
last = tick
# print bar.volume
if bar: #写入bar
dict_ = bar.__dict__
if dict_.has_key('_id'):
del dict_['_id']
collbar_1m.insert_one(dict_)
print 'Write {} Bar:'.format(nmin), bar.__dict__
return bar
def loadSetting(self):
"""载入设置"""
with open(self.settingFileName) as f:
drSetting = json.load(f)
# 如果working设为False则不启动行情记录功能
working = drSetting['working']
if not working:
return
if 'tick' in drSetting:
l = drSetting['tick']
for setting in l:
symbol = setting[0]
vtSymbol = symbol
req = VtSubscribeReq()
req.symbol = setting[0]
# 针对LTS和IB接口，订阅行情需要交易所代码
if len(setting) >= 3:
req.exchange = setting[2]
vtSymbol = '.'.join([symbol, req.exchange])
# 针对IB接口，订阅行情需要货币和产品类型
if len(setting) >= 5:
req.currency = setting[3]
req.productClass = setting[4]
self.mainEngine.subscribe(req, setting[1])
tick = VtTickData() # 该tick实例可以用于缓存部分数据（目前未使用）
self.tickDict[vtSymbol] = tick
if 'bar' in drSetting:
l = drSetting['bar']
for setting in l:
symbol = setting[0]
vtSymbol = symbol
req = VtSubscribeReq()
req.symbol = symbol
if len(setting) >= 3:
req.exchange = setting[2]
vtSymbol = '.'.join([symbol, req.exchange])
if len(setting) >= 5:
req.currency = setting[3]
req.productClass = setting[4]
self.mainEngine.subscribe(req, setting[1])
bar = VtBarData()
self.barDict[vtSymbol] = bar
if 'active' in drSetting:
d = drSetting['active'] # 各品种的主力合约
# 注意这里的vtSymbol对于IB和LTS接口，应该后缀.交易所
for activeSymbol, vtSymbol in d.items():
self.activeSymbolDict[vtSymbol] = activeSymbol
# 启动数据插入线程
self.start()
# 注册事件监听
self.registerEvent()
def loadSetting(self):
"""加载配置"""
with open(self.settingFileName, 'r') as f:
drSetting = json.load(f)
# 如果working设为False则不启动行情记录功能
working = drSetting['working']
if not working:
return
if 'tick' in drSetting:
l = drSetting['tick']
for setting in l:
symbol = setting[0]
vtSymbol = symbol
req = VtSubscribeReq()
req.symbol = setting[0]
# 针对LTS和IB接口，订阅行情需要交易所代码
if len(setting) >= 3:
req.exchange = setting[2]
vtSymbol = '.'.join([symbol, req.exchange])
# 针对IB接口，订阅行情需要货币和产品类型
if len(setting) >= 5:
req.currency = setting[3]
req.productClass = setting[4]
self.mainEngine.subscribe(req, setting[1])
tick = VtTickData() # 该tick实例可以用于缓存部分数据（目前未使用）
self.tickDict[vtSymbol] = tick
# 保存到配置字典中
if vtSymbol not in self.settingDict:
d = {
'symbol': symbol,
'gateway': setting[1],
'tick': True
}
self.settingDict[vtSymbol] = d
else:
d = self.settingDict[vtSymbol]
d['tick'] = True
if 'bar' in drSetting:
l = drSetting['bar']
for setting in l:
symbol = setting[0]
vtSymbol = symbol
req = VtSubscribeReq()
req.symbol = symbol
if len(setting) >= 3:
req.exchange = setting[2]
vtSymbol = '.'.join([symbol, req.exchange])
if len(setting) >= 5:
req.currency = setting[3]
req.productClass = setting[4]
self.mainEngine.subscribe(req, setting[1])
bar = VtBarData()
self.barDict[vtSymbol] = bar
# 保存到配置字典中
if vtSymbol not in self.settingDict:
d = {
'symbol': symbol,
'gateway': setting[1],
'bar': True
}
self.settingDict[vtSymbol] = d
else:
d = self.settingDict[vtSymbol]
d['bar'] = True
if 'active' in drSetting:
d = drSetting['active']
# 注意这里的vtSymbol对于IB和LTS接口，应该后缀.交易所
for activeSymbol, vtSymbol in list(d.items()):
self.activeSymbolDict[vtSymbol] = activeSymbol
# 保存到配置字典中
if vtSymbol not in self.settingDict:
d = {
'symbol': symbol,
'gateway': setting[1],
'active': True
}
self.settingDict[vtSymbol] = d
else:
d = self.settingDict[vtSymbol]
d['active'] = True
def processLine(self, line):
historyData = line.split(',')
#historyDataLen = len(historyData)
symbol = historyData[2]
#print 'processLine, symbol:' + symbol
#从list转化为tick对象
historytick = VtTickData()
historytick._id = historyData[0]
historytick.gatewayName = 'CTP'
historytick.symbol = symbol
historytick.TradingDay = historyData[1]
historytick.exchange = historyData[3]
historytick.vtSymbol = historytick.symbol # '.'.join([tick.symbol, tick.exchange])
historytick.lastPrice = self.convertFloatZero(historyData[5])
#lastVolume
historytick.volume = historyData[12]
historytick.openInterest = historyData[14]
UpdateMillisec = int(historyData[20])
historytick.time = '.'.join([historyData[19], str(UpdateMillisec/ 100)])
historytick.date = historyData[42]
historytick.datetime = datetime.strptime(' '.join([historytick.date, historytick.time]), '%Y%m%d %H:%M:%S.%f')
historytick.openPrice = self.convertFloatZero(historyData[9])
historytick.highPrice = self.convertFloatZero(historyData[10])
historytick.lowPrice = self.convertFloatZero(historyData[11])
historytick.preClosePrice = self.convertFloatZero(historyData[12])
historytick.ClosePrice = self.convertFloatZero(historyData[15])
historytick.SettlementPrice = self.convertFloatZero(historyData[16])
historytick.upperLimit = self.convertFloatZero(historyData[17])
historytick.lowerLimit = self.convertFloatZero(historyData[18])
# CTP只有一档行情
historytick.bidPrice1 = self.convertFloatZero(historyData[21])
historytick.bidVolume1 = historyData[22]
historytick.askPrice1 = self.convertFloatZero(historyData[23])
historytick.askVolume1 = historyData[24]
historytick.AveragePrice = self.convertFloatZero(historyData[41])
return historytick
def loadTickData(self, tick):
tickData = VtTickData()
tickData.lastPrice = float(tick['lastPrice'])
tickData.date = tick['date']
tickData.time = tick['time']
tickData.datetime = datetime.strptime(
' '.join([tick['date'], tick['time']]), '%Y%m%d %H:%M:%S.%f')
tickData.volume = int(tick['volume'])
tickData.vtSymbol = tick['vtSymbol']
tickData.symbol = tick['symbol']
tickData.exchange = tick['exchange']
tickData.openInterest = int(tick['openInterest'])
return tickData
def loadTickData(self, tick):
tickData = VtTickData()
#tickData.lastPrice = float(tick['lastPrice'])
#要处理小数位数，只保留一位小数
#线上录得数据写文件，比如原始数据是个整数3967，写到文件里的会是3967.000000000001
tickData.lastPrice = round(float(tick['lastPrice']), 1)
tickData.date = tick['date']
tickData.time = tick['time']
tickData.datetime = datetime.strptime(' '.join([tick['date'], tick['time']]), '%Y%m%d %H:%M:%S.%f')
tickData.volume = int(tick['volume'])
tickData.vtSymbol = tick['vtSymbol']
tickData.symbol = tick['symbol']
tickData.exchange = tick['exchange']
tickData.TradingDay = tick['TradingDay']
#tickData.openInterest = int(tick['openInterest'])
#防止持仓量可能带小数
tickData.openInterest = int(float(tick['openInterest']))
return tickData
def startWork(self):
for root, subdirs, files in os.walk(self.walkingDir):
for theFile in files:
currentDate = None
currentSymbol = ''
count = 0
# 确定合约【由文件夹名称决定，因为主力合约只能在文件夹名称中确定】
if '\\' in root:
currentSymbol = root.split('\\')[-1]
currentSymbol = currentSymbol
# 排除不合法文件
if theFile.startswith('.'):
continue
# 确定日期
if '.' in theFile:
pos = theFile.index('.')
dateStr = theFile[:pos]
currentDate = datetime.datetime.strptime(dateStr, '%Y%m%d')
# 合约不存在
if not currentSymbol:
print('*' * 6)
print(u'没有找到合约')
print('*' * 6)
exit(0)
# 检查是否有数据库记录
if currentDate and (not self.needUpdateChecking(
currentSymbol, currentDate)):
continue
if '.csv' in theFile:
# 读取文件
filePath = root + '\\' + theFile
with open(filePath, 'r') as f:
reader = csv.DictReader(f)
# 开始导入数据
startTime = time()
for row in reader:
count += 1
if count == 1:
print('=' * 6, currentSymbol, ' ',
currentDate.date(), '=' * 6)
# 过滤无效数据
timeStr = row['time']
theLen = len(timeStr)
if theLen < 9:
sub = 9 - theLen
timeStr = '0' * sub + timeStr
'''
hour = int(timeStr[0:2])
minute = int(timeStr[2:4])
second = int(timeStr[4:6])
microSecond = int(timeStr[6:9])
if hour < 0 or hour > 59 or minute < 0 or minute > 59 or second < 0 or second > 59 or microSecond < 0 or microSecond > 999:
print u'删除时间格式错误数据：%s' % timeStr
continue
'''
try:
tickTime = datetime.datetime.strptime(
timeStr, '%H%M%S%f')
except Exception:
print(u'时间转换异常：%s' % timeStr)
raise
exit(0)
t = tickTime.time()
fakeData = True
if self.dataType == 1:
# 商品期货
if self.night == 0:
# 无夜盘交易
if ((MORNING_START_CF <= t <
MORNING_REST_CF)
or (MORNING_RESTART_CF <= t <
MORNING_END_CF)
or (AFTERNOON_START_CF <= t <
AFTERNOON_END_CF)):
fakeData = False
elif self.night == 1:
# 夜间收盘
if ((MORNING_START_CF <= t <
MORNING_REST_CF)
or (MORNING_RESTART_CF <= t <
MORNING_END_CF)
or (AFTERNOON_START_CF <= t <
AFTERNOON_END_CF)
or (NIGHT_START_CF <= t <
NIGHT_END_CF_N)):
fakeData = False
elif self.night == 2:
# 凌晨收盘
if ((MORNING_START_CF <= t <
MORNING_REST_CF)
or (MORNING_RESTART_CF <= t <
MORNING_END_CF)
or (AFTERNOON_START_CF <= t <
AFTERNOON_END_CF)
or (NIGHT_START_CF <= t)
or (NIGHT_END_CF_M > t)):
fakeData = False
elif self.dataType == 2:
# 股指期货
if ((MORNING_START_SF <= t < MORNING_END_SF)
or (AFTERNOON_START_SF <= t <
AFTERNOON_END_SF)):
fakeData = False
theDate = row['calendarday']
theTime = tickTime.strftime('%H:%M:%S.%f')
theDatetime = datetime.datetime.strptime(
theDate + ' ' + theTime, '%Y%m%d %H:%M:%S.%f')
# 数据库collection
collection = self.db[currentSymbol]
collection.create_index('datetime')
if fakeData:
print(t)
print(u'【剔除无效数据】')
# 删除已保存的数据库数据
flt = {'datetime': theDatetime}
cursor = collection.find(flt)
if cursor.count():
print(u'【剔除无效‘数据库’数据】')
collection.delete_many(flt)
continue
# 创建VtTickData对象
tick = VtTickData()
tick.vtSymbol = row['instrument']
tick.lastPrice = float(row['lastp'])
tick.volume = int(row['volume'])
tick.openInterest = int(row['openinterest'])
tick.askPrice1 = float(row['ask1'])
tick.askVolume1 = int(row['asksz1'])
tick.bidPrice1 = float(row['bid1'])
tick.bidVolume1 = int(row['bidsz1'])
tick.date = theDate
tick.time = theTime
tick.datetime = theDatetime
tick.tradingDay = row['tradingday']
# 保存tick到数据库
collection.update_many({'datetime': tick.datetime},
{'$set': tick.__dict__},
upsert=True)
# 打印进程
if count:
sub = time() - startTime
print(u'用时：', sub, 's')
print(u'数据量：', count, '\n')
# 保存记录
if currentDate and count:
self.updateHistory(currentSymbol, currentDate)
print(u'所有数据导入完成')
def load13Csv(fileName, dbName, symbol):
"""将TB极速版导出的csv格式的历史分钟数据插入到Mongo数据库中"""
import csv
count = 0
start = time()
print u'开始读取CSV文件%s中的数据插入到%s的%s中' % (fileName, dbName, symbol)
# 锁定集合，并创建索引
client = pymongo.MongoClient(globalSetting['mongoHost'],
globalSetting['mongoPort'])
collection = client[dbName][symbol]
collection.ensure_index([('datetime', pymongo.ASCENDING)], unique=True)
# 读取数据和插入到数据库
reader = csv.reader(file(fileName, 'r'))
lasttime = None
for d in reader:
tick = VtTickData()
tick.vtSymbol = symbol
tick.symbol = symbol
tick.lastPrice = float(d[2])
tick.volume = int(d[6])
tick.askPrice1 = float(d[3])
tick.bidPrice1 = float(d[4])
count += 1
zhongjian = datetime.strptime(d[0], '%Y-%m-%d')
tick.date = zhongjian.strftime('%Y%m%d')
if d[1] == lasttime:
tick.time = d[1] + "." + "500000"
else:
tick.time = d[1] + "." + "000000"
lasttime = d[1]
tick.datetime = datetime.strptime(tick.date + ' ' + tick.time,
'%Y%m%d %H:%M:%S.%f')
flt = {'datetime': tick.datetime}
collection.update_one(flt, {'$set': tick.__dict__}, upsert=True)
print u'插入完毕，耗时：%s插入数量' % (time() - start), count
def mongodbRowToTick(row_tick):
tickData = VtTickData()
tickData.lastPrice = float(row_tick['lastPrice'])
tickData.date = row_tick['date']
tickData.time = row_tick['time']
tickData.datetime = datetime.strptime(
' '.join([row_tick['date'], row_tick['time']]), '%Y%m%d %H:%M:%S.%f')
tickData.volume = int(row_tick['volume'])
tickData.vtSymbol = row_tick['vtSymbol']
tickData.symbol = row_tick['symbol']
tickData.exchange = row_tick['exchange']
tickData.TradingDay = row_tick['TradingDay']
tickData.openInterest = int(row_tick['openInterest'])
return tickData
def generateVtTick(row, symbol):
"""生成K线"""
tick = VtTickData()
tick.symbol = symbol
tick.vtSymbol = symbol
tick.lastPrice = row['last']
tick.volume = row['volume']
tick.openInterest = row['open_interest']
tick.datetime = row.name
tick.openPrice = row['open']
tick.highPrice = row['high']
tick.lowPrice = row['low']
tick.preClosePrice = row['prev_close']
tick.upperLimit = row['limit_up']
tick.lowerLimit = row['limit_down']
tick.bidPrice1 = row['b1']
tick.bidPrice2 = row['b2']
tick.bidPrice3 = row['b3']
tick.bidPrice4 = row['b4']
tick.bidPrice5 = row['b5']
tick.bidVolume1 = row['b1_v']
tick.bidVolume2 = row['b2_v']
tick.bidVolume3 = row['b3_v']
tick.bidVolume4 = row['b4_v']
tick.bidVolume5 = row['b5_v']
tick.askPrice1 = row['a1']
tick.askPrice2 = row['a2']
tick.askPrice3 = row['a3']
tick.askPrice4 = row['a4']
tick.askPrice5 = row['a5']
tick.askVolume1 = row['a1_v']
tick.askVolume2 = row['a2_v']
tick.askVolume3 = row['a3_v']
tick.askVolume4 = row['a4_v']
tick.askVolume5 = row['a5_v']
return tick
def generateVtTick(symbol, d):
tick = VtTickData()
tick.symbol = symbol
tick.VtSymbol = symbol
def make_day_bar(symbol,date,drop =True,product=ProductClass.Future):
"""
计算日线
"""
from logging import getLogger
if isinstance(date,str):
date = parse(date)
date = date.replace(hour=0,minute=0,second=0,microsecond=0)
days = get_timespace_of_trade_day(date)
bar = None
if not days:
getLogger().info("day span is None")
return None
t1,t2 = days
conn = mongodb_conn
dbname = CTP_TICK_DB
coll = conn[dbname][symbol]
rs = coll.find({'datetime':{'$gte':t1,'$lte':t2},'flag':0}).sort( (['datetime',pymongo.ASCENDING],['seq',pymongo.ASCENDING]))
print symbol, t1, '-->', t2, ' count:',rs.count()
dbname = CTP_BAR_DB
collbar_1m = conn[dbname.format('1d')][symbol]
# 删除已存在 1m bar
if drop:
collbar_1m.remove({'datetime':date})
# 注意，记录的日期是日盘交易的日期
if rs.count() == 0: # 分钟内无tick数据 ，将连续之前的bar
# if not prev_bar: #查找前一个bar
prev_bar = None
r = collbar_1m.find({'datetime':{'$lt':t1},'flag':0}).sort('datetime',pymongo.DESCENDING).limit(1)
if r.count() :
prev_bar = VtBarData()
prev_bar.__dict__ = r[0]
if prev_bar:
prev_bar.datetime = date
prev_bar.date = prev_bar.datetime.strftime('%Y%m%d')
prev_bar.time = prev_bar.datetime.strftime('%H:%M:%S.%f')
prev_bar.high = prev_bar.close
prev_bar.low = prev_bar.close
prev_bar.open = prev_bar.close
prev_bar.volume = 0
bar = prev_bar
else: # 计算当前分钟内的bar
bar = VtBarData()
tick = VtTickData()
tick.__dict__ = rs[0]
bar.datetime = date
bar.date = bar.datetime.strftime('%Y%m%d')
bar.time = bar.datetime.strftime('%H:%M:%S.%f')
bar.vtSymbol = tick.vtSymbol
bar.symbol = tick.symbol
bar.exchange = tick.exchange
bar.open = tick.lastPrice
bar.high = tick.lastPrice
bar.low = tick.lastPrice
last = None
# 找到前一个k线最后一个tick作为last
rs = coll.find({'datetime': {'$lt': t1}, 'flag': 0}).sort(
(['datetime', pymongo.DESCENDING], ['seq', pymongo.DESCENDING])).limit(1)
if rs.count():
last = VtTickData()
last.__dict__ = rs[0]
for r in rs:
tick = VtTickData()
tick.__dict__ = r
bar.high = max(bar.high, tick.lastPrice)
bar.low = min(bar.low, tick.lastPrice)
bar.close = tick.lastPrice
bar.openInterest = tick.openInterest
if last:
bar.volume += (tick.volume - last.volume)
last = tick
if bar: #写入bar
dict_ = bar.__dict__
if dict_.has_key('_id'):
del dict_['_id']
collbar_1m.insert_one(dict_)
print 'Write {} Bar:'.format(str(date.date())),bar.__dict__
return bar
def make_min_bar(symbol,t,nmin='1m',leftMargin=0,rightMargin=0,drop =True,product=ProductClass.Future):
"""
开始计算指定开始分钟的 bar
:drop 是否删除存在的分钟 bar
1. 指定的分钟内没有tick数据
取前一根有效的bar
如果前无bar，则忽略本次bar计算
"""
if nmin not in TimeDuration.SCALES.keys():
print 'Error: paramter nmin:{} invalid.'.format(nmin)
return
mins = TimeDuration.SCALES[nmin]/TimeDuration.MINUTE
t1 = t.replace(second=0,microsecond=0)
t2 = t1 + TimeDelta(minutes=mins)
conn = mongodb_conn
dbname = CTP_TICK_DB
coll = conn[dbname][symbol]
p1 = t1 - TimeDelta(minutes=leftMargin)
p2 = t2 + TimeDelta(minutes=rightMargin)
rs = coll.find({'datetime':{'$gte':p1,'$lt':p2},'flag':0}).sort( (['datetime',pymongo.ASCENDING],['seq',pymongo.ASCENDING]))
print symbol, p1, '-->', p2, ' count:',rs.count()
dbname = CTP_BAR_DB
collbar_1m = conn[dbname.format(nmin)][symbol]
bar = None
# 删除已存在 1m bar
if drop:
collbar_1m.remove({'datetime':t1})
if rs.count() == 0: # 分钟内无tick数据 ，将连续之前的bar
# if not prev_bar: #查找前一个bar
prev_bar = None
r = collbar_1m.find({'datetime':{'$lt':t1},'flag':0}).sort('datetime',pymongo.DESCENDING).limit(1)
if r.count() :
prev_bar = VtBarData()
prev_bar.__dict__ = r[0]
if prev_bar:
prev_bar.datetime = t1
prev_bar.date = prev_bar.datetime.strftime('%Y%m%d')
prev_bar.time = prev_bar.datetime.strftime('%H:%M:%S.%f')
prev_bar.high = prev_bar.close
prev_bar.low = prev_bar.close
prev_bar.open = prev_bar.close
prev_bar.volume = 0
bar = prev_bar
else: # 计算当前分钟内的bar
bar = VtBarData()
tick = VtTickData()
tick.__dict__ = rs[0]
bar.datetime = t1
bar.date = bar.datetime.strftime('%Y%m%d')
bar.time = bar.datetime.strftime('%H:%M:%S.%f')
bar.vtSymbol = tick.vtSymbol
bar.symbol = tick.symbol
bar.exchange = tick.exchange
bar.open = tick.lastPrice
bar.high = tick.lastPrice
bar.low = tick.lastPrice
last = None
#找到前一个k线最后一个tick作为last
rs = coll.find({'datetime': {'$lt': p1}, 'flag': 0}).sort((['datetime', pymongo.DESCENDING], ['seq', pymongo.DESCENDING])).limit(1)
if rs.count():
last = VtTickData()
last.__dict__ = rs[0]
# --
for r in rs:
tick = VtTickData()
tick.__dict__ = r
bar.high = max(bar.high, tick.lastPrice)
bar.low = min(bar.low, tick.lastPrice)
bar.close = tick.lastPrice
bar.openInterest = tick.openInterest
if last:
bar.volume += (tick.volume - last.volume)
last = tick
if bar: #写入bar
dict_ = bar.__dict__
if dict_.has_key('_id'):
del dict_['_id']
collbar_1m.insert_one(dict_)
print 'Write {} Bar:'.format(nmin),bar.__dict__
return bar
def generateVtTick(row, symbol):
"""生成K线"""
tick = VtTickData()
tick.symbol = symbol
tick.vtSymbol = symbol
tick.lastPrice = row['last']
tick.volume = row['volume']
tick.openInterest = row['open_interest']
tick.datetime = row.name
tick.openPrice = row['open']
tick.highPrice = row['high']
tick.lowPrice = row['low']
tick.preClosePrice = row['prev_close']
tick.upperLimit = row['limit_up']
tick.lowerLimit = row['limit_down']
tick.bidPrice1 = row['b1']
tick.bidPrice2 = row['b2']
tick.bidPrice3 = row['b3']
tick.bidPrice4 = row['b4']
tick.bidPrice5 = row['b5']
tick.bidVolume1 = row['b1_v']
tick.bidVolume2 = row['b2_v']
tick.bidVolume3 = row['b3_v']
tick.bidVolume4 = row['b4_v']
tick.bidVolume5 = row['b5_v']
tick.askPrice1 = row['a1']
tick.askPrice2 = row['a2']
tick.askPrice3 = row['a3']
tick.askPrice4 = row['a4']
tick.askPrice5 = row['a5']
tick.askVolume1 = row['a1_v']
tick.askVolume2 = row['a2_v']
tick.askVolume3 = row['a3_v']
tick.askVolume4 = row['a4_v']
tick.askVolume5 = row['a5_v']
return tick
def add_gap_ticks(self):
"""
补充缺失的分时数据
:return:
"""
ds = DataSource()
for vtSymbol in self.renkoDict.keys():
renkobar_list = self.renkoDict.get(vtSymbol, [])
cache_bars = None
cache_start_date = None
cache_end_date = None
for renko_bar in renkobar_list:
# 通过mongo获取最新一个Renko bar的数据日期close时间
last_renko_dt = self.get_last_datetime(renko_bar.name)
# 根据日期+vtSymbol，像datasource获取分钟数据，以close价格，转化为tick，推送到renko_bar中
if last_renko_dt is not None:
start_date = last_renko_dt.strftime('%Y-%m-%d')
else:
start_date = (datetime.now() -
timedelta(days=90)).strftime('%Y-%m-%d')
end_date = (datetime.now() +
timedelta(days=5)).strftime('%Y-%m-%d')
self.writeDrLog(u'从datasource获取{}数据,开始日期:{}'.format(
vtSymbol, start_date))
if cache_bars is None or cache_start_date != start_date or cache_end_date != end_date:
fields = [
'open', 'close', 'high', 'low', 'volume',
'open_interest', 'limit_up', 'limit_down',
'trading_date'
]
cache_bars = ds.get_price(order_book_id=vtSymbol,
start_date=start_date,
end_date=end_date,
frequency='1m',
fields=fields)
cache_start_date = start_date
cache_end_date = end_date
if cache_bars is not None:
total = len(cache_bars)
self.writeDrLog(u'一共获取{}条{} 1分钟数据'.format(
total, vtSymbol))
if cache_bars is not None:
self.writeDrLog(u'推送分时数据tick:{}到:{}'.format(
vtSymbol, renko_bar.name))
for idx in cache_bars.index:
row = cache_bars.loc[idx]
tick = VtTickData()
tick.vtSymbol = vtSymbol
tick.symbol = vtSymbol
last_bar_dt = datetime.strptime(
str(idx), '%Y-%m-%d %H:%M:00')
tick.datetime = last_bar_dt - timedelta(minutes=1)
tick.date = tick.datetime.strftime('%Y-%m-%d')
tick.time = tick.datetime.strftime('%H:%M:00')
if tick.datetime.hour >= 21:
if tick.datetime.isoweekday() == 5:
# 星期五=》星期一
tick.tradingDay = (
tick.datetime +
timedelta(days=3)).strftime('%Y-%m-%d')
else:
# 第二天
tick.tradingDay = (
tick.datetime +
timedelta(days=1)).strftime('%Y-%m-%d')
elif tick.datetime.hour < 8 and tick.datetime.isoweekday(
) == 6:
# 星期六=>星期一
tick.tradingDay = (
tick.datetime +
timedelta(days=2)).strftime('%Y-%m-%d')
else:
tick.tradingDay = tick.date
tick.upperLimit = float(row['limit_up'])
tick.lowerLimit = float(row['limit_down'])
tick.lastPrice = float(row['close'])
tick.askPrice1 = float(row['close'])
tick.bidPrice1 = float(row['close'])
tick.volume = int(row['volume'])
tick.askVolume1 = tick.volume
tick.bidVolume1 = tick.volume
if last_renko_dt is not None and tick.datetime <= last_renko_dt:
continue
renko_bar.onTick(tick)
