def updateTick(self, tick):
"""TICK更新"""
newMinute = False # 默认不是新的一分钟
# 尚未创建对象
if not self.bar:
#self.bar = VtBarData()
self.bar = CtaBarData()
newMinute = True
# 新的一分钟
elif self.bar.datetime.minute != tick.datetime.minute:
# 生成上一分钟K线的时间戳
self.bar.datetime = self.bar.datetime.replace(
second=0, microsecond=0) # 将秒和微秒设为0
self.bar.date = self.bar.datetime.strftime('%Y-%m-%d')
self.bar.time = self.bar.datetime.strftime('%H:%M:%S')
self.bar.tradingDay = self.bar.date
# 推送已经结束的上一分钟K线
self.onBar(self.bar)
print(u'BarGenerate:Onbar:{},{},o:{},h:{},l:{},c:{}'.format(
self.bar.vtSymbol, self.bar.datetime, self.bar.open,
self.bar.high, self.bar.low, self.bar.close))
# 创建新的K线对象
#self.bar = VtBarData()
self.bar = CtaBarData()
newMinute = True
# 初始化新一分钟的K线数据
if newMinute:
self.bar.vtSymbol = tick.vtSymbol
self.bar.symbol = tick.symbol
self.bar.exchange = tick.exchange
self.bar.open = tick.lastPrice
self.bar.high = tick.lastPrice
self.bar.low = tick.lastPrice
# 累加更新老一分钟的K线数据
else:
self.bar.high = max(self.bar.high, tick.lastPrice)
self.bar.low = min(self.bar.low, tick.lastPrice)
# 通用更新部分
self.bar.close = tick.lastPrice
self.bar.datetime = tick.datetime
self.bar.openInterest = tick.openInterest
if self.lastTick:
volumeChange = tick.volume - self.lastTick.volume # 当前K线内的成交量
self.bar.volume += max(
volumeChange, 0) # 避免夜盘开盘lastTick.volume为昨日收盘数据，导致成交量变化为负的情况
# 缓存Tick
self.lastTick = tick
def get_bars(self,
symbol,
period,
callback,
bar_is_completed=False,
bar_freq=1,
start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
if symbol not in SYMBOL_LIST:
self.writeError(u'{} 合约{}不在下载清单中'.format(datetime.now(), symbol))
return False, ret_bars
if period not in PERIOD_MAPPING:
self.writeError(u'{} 周期{}不在下载清单中'.format(datetime.now(), period))
return False, ret_bars
if self.client is None:
return False, ret_bars
binance_symbol = symbol.upper().replace('_', '')
# 转换一下
binance_symbol = symbolFromOtherExchangesToBinance(binance_symbol)
binance_period = PERIOD_MAPPING.get(period)
self.writeLog('{}开始下载binance:{} {}数据.'.format(datetime.now(),
binance_symbol,
binance_period))
bars = []
try:
bars = self.client.get_klines(symbol=binance_symbol,
interval=binance_period)
bar_len = len(bars)
for i, bar in enumerate(bars):
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000)
utcdatetime = datetime.utcfromtimestamp(bar[0] / 1e3)
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
# 币安的交易日，是按照utc来计算的
add_bar.tradingDay = utcdatetime.strftime('%Y-%m-%d')
add_bar.open = float(bar[1])
add_bar.high = float(bar[2])
add_bar.low = float(bar[3])
add_bar.close = float(bar[4])
add_bar.volume = float(bar[5])
ret_bars.append(add_bar)
except Exception as ex:
self.writeError(
'error when convert bar:{},ex:{},t:{}'.format(
bar, str(ex), traceback.format_exc()))
return False, ret_bars
if start_dt is not None and bar.datetime < start_dt:
continue
if callback is not None:
# 最后一个bar，可能是不完整的，强制修改
if i == bar_len - 1 and bar_is_completed:
# 根据秒数算的话，要+1，例如13:31,freq=31，第31根bar
freq = int((datetime.now() -
add_bar.datetime).total_seconds() / 60) + 1
callback(add_bar, False, freq)
else:
callback(add_bar, bar_is_completed, bar_freq)
return True, ret_bars
except Exception as ex:
self.writeError('exception in get:{},{},{}'.format(
binance_symbol, str(ex), traceback.format_exc()))
return False, ret_bars
def getMinBars(self, symbol, callback):
bars = []
try:
path = 'hisminbar'
params = {'id': symbol, 'num': 172, 'period': '15m'}
data = self.getData(path, params)
if not data:
return False
barList = []
l = data.split(';')
for barStr in l:
# 过滤某些空数据
if ',' not in barStr:
continue
barData = barStr.split(',')
d = {
'symbol': barData[0],
# 'date': barData[1], # trading day
'tradingDay': barData[1],
'minute': barData[2],
'time': barData[2],
'open': float(barData[3]),
'high': float(barData[4]),
'low': float(barData[5]),
'close': float(barData[6]),
'volume': int(barData[7]),
'openInterest': int(float(barData[8])),
'date': barData[9] # natural day
}
barList.append(d)
barList.reverse()
for item in barList:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
bar.tradingDay = item['tradingDay']
bar.datetime = datetime.strptime(
u'{0}-{1}-{2} {3}:{4}:00'.format(item['date'][0:4],
item['date'][4:6],
item['date'][6:8],
item['minute'][0:2],
item['minute'][2:4]),
'%Y-%m-%d %H:%M:00')
bar.time = item['time']
bar.open = item['open']
bar.high = item['high']
bar.low = item['low']
bar.close = item['close']
bar.volume = item['volume']
bar.date = item['date']
if bar.datetime.hour == 10 and bar.datetime.minute == 30:
continue
# bar.datetime = bar.datetime - timedelta(seconds=5 * 60)
callback(bar, bar_is_completed=False, bar_freq=15)
# print(u'{0}, {1}, {2}, {3}, {4}'.format(bar.datetime, bar.open, bar.high, bar.low, bar.close))
# bars.append(bar)
# print('*' * 20 + 'bars---' + '*' * 20)
# print(bars)
# if len(bars) > 0:
# for bar in bars:
# callback(bar)
# bars = []
return True
# else:
# self.strategy.writeCtaLog(u'从shcifo读取分钟数据失败')
# return False
except Exception as e:
self.strategy.writeCtaLog(u'加载shcifo历史分钟数据失败：' + str(e))
return False
def get_bars(self, symbol, period, callback, bar_is_completed=False,bar_freq=1, start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
if symbol not in symbol_list:
self.strategy.writeCtaError(u'{} {}不在下载清单中'.format(datetime.now(), symbol))
return False, ret_bars
if period not in PERIOD_MAPPING:
self.strategy.writeCtaError(u'{} 周期{}不在下载清单中'.format(datetime.now(), period))
return False, ret_bars
self.strategy.writeCtaLog('{}开始下载:{} {}数据'.format(datetime.now(), symbol, period))
fcoin_period = PERIOD_MAPPING.get(period)
fcoin_symbol = symbol.lower().replace('_' , '')
try:
status,response = self.fcoin.get_candle(resolution=fcoin_period,symbol=fcoin_symbol,limit=200)
if not status:
return False, ret_bars
bars = response.get('data')
bars.reverse()
#{'seq': 7636160700002, 'low': 6747.96, 'open': 6752.49, 'high': 6753.37, 'base_vol': 356.859787045,
# 'quote_vol': 2409051.1568048545, 'id': 1529566740, 'close': 6750.72, 'count': 3673}
for i, bar in enumerate(bars):
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.fromtimestamp(bar.get('id',datetime.now()) )
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.open = float(bar.get('open',0.0))
add_bar.high = float(bar.get('high',0.0))
add_bar.low = float(bar.get('low',0.0))
add_bar.close = float(bar.get('close',0.0))
add_bar.volume = float(bar.get('base_vol',0.0))
except Exception as ex:
self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc()))
return False, ret_bars
if start_dt is not None and bar.datetime < start_dt:
continue
ret_bars.append(add_bar)
if callback is not None:
callback(add_bar, bar_is_completed, bar_freq)
return True, ret_bars
except Exception as ex:
self.strategy.writeCtaError('exception in get bar:{},{}'.format(str(ex), traceback.format_exc()))
return False
def get_bars(self,
symbol,
period,
callback,
bar_is_completed=False,
bar_freq=1,
start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
if symbol not in SYMBOL_LIST:
self.strategy.writeCtaError(u'{} 合约{}不在下载清单中'.format(
datetime.now(), symbol))
return False, ret_bars
if period not in PERIOD_MAPPING:
self.strategy.writeCtaError(u'{} 周期{}不在下载清单中'.format(
datetime.now(), period))
return False, ret_bars
if self.client is None:
return False, ret_bars
binance_symbol = symbol.upper().replace('_', '')
binance_period = PERIOD_MAPPING.get(period)
self.strategy.writeCtaLog('{}开始下载binance:{} {}数据.'.format(
datetime.now(), binance_symbol, binance_period))
try:
bars = self.client.get_klines(symbol=binance_symbol,
interval=binance_period)
for i, bar in enumerate(bars):
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000)
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.open = float(bar[1])
add_bar.high = float(bar[2])
add_bar.low = float(bar[3])
add_bar.close = float(bar[4])
add_bar.volume = float(bar[5])
except Exception as ex:
self.strategy.writeCtaError(
'error when convert bar:{},ex:{},t:{}'.format(
bar, str(ex), traceback.format_exc()))
return False
if start_dt is not None and bar.datetime < start_dt:
continue
ret_bars.append(add_bar)
if callback is not None:
callback(add_bar, bar_is_completed, bar_freq)
return True, ret_bars
except Exception as ex:
self.strategy.writeCtaError('exception in get:{},{},{}'.format(
binance_symbol, str(ex), traceback.format_exc()))
return False, ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False,bar_freq=1, start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
if symbol not in SYMBOL_LIST:
self.strategy.writeCtaError(u'{} 合约{}不在下载清单中'.format(datetime.now(), symbol))
return False
if period not in PERIOD_MAPPING:
self.strategy.writeCtaError(u'{} 周期{}不在下载清单中'.format(datetime.now(), period))
return False
if self.client is None:
return False
binance_symbol = symbol.upper().replace('_' , '')
binance_period = PERIOD_MAPPING.get(period)
self.strategy.writeCtaLog('{}开始下载binance:{} {}数据.'.format(datetime.now(), binance_symbol, binance_period))
bars = []
try:
bars = self.client.get_klines(symbol=binance_symbol, interval=binance_period)
for i, bar in enumerate(bars):
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000)
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.open = float(bar[1])
add_bar.high = float(bar[2])
add_bar.low = float(bar[3])
add_bar.close = float(bar[4])
add_bar.volume = float(bar[5])
except Exception as ex:
self.strategy.writeCtaError(
'error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc()))
return False
if start_dt is not None and bar.datetime < start_dt:
continue
if callback is not None:
callback(add_bar, bar_is_completed, bar_freq)
return True
except Exception as ex:
self.strategy.writeCtaError('exception in get:{},{},{}'.format(binance_symbol,str(ex), traceback.format_exc()))
return False
def get_bars(self, symbol, period, callback, bar_is_completed=False,bar_freq=1, start_dt=None):
"""
返回k线数据
symbol：合约b tc:next_week:10
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
if ':' not in symbol:
self.strategy.writeCtaError(u'{} {}格式需要包含合约类型,如:btc:next_week:10'.format(datetime.now(), symbol))
return False, ret_bars
s = symbol.split(':')
symbol_pair, contract_type = s[0],s[1]
if not symbol_pair.endswith('_usd'):
symbol_pair += '_usd'
if symbol_pair not in symbol_list:
self.strategy.writeCtaError(u'{} {}不在下载清单中'.format(datetime.now(), symbol_pair))
return False, ret_bars
url = u'https://www.okex.com/api/v1/future_kline.do?symbol={}&type={}&contract_type={}'.format(symbol_pair, period,contract_type)
self.strategy.writeCtaLog('{}开始下载:{} {}数据.URL:{}'.format(datetime.now(), symbol, period, url))
bars = []
content = None
try:
content = self.session.get(url).content.decode('gbk')
bars = execjs.eval(content)
except Exception as ex:
self.strategy.writeCtaError('exception in get:{},{},{}'.format(url,str(ex), traceback.format_exc()))
return False, ret_bars
for i, bar in enumerate(bars):
if len(bar) < 5:
self.strategy.writeCtaError('error when import bar:{}'.format(bar))
return False
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000)
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.open = float(bar[1])
add_bar.high = float(bar[2])
add_bar.low = float(bar[3])
add_bar.close = float(bar[4])
add_bar.volume = float(bar[6]) # 这里：5 是交易量，6是交易量转化BTC或LTC数量
except Exception as ex:
self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc()))
return False, ret_bars
if start_dt is not None and bar.datetime < start_dt:
continue
ret_bars.append(add_bar)
if callback is not None:
callback(add_bar, bar_is_completed, bar_freq)
return True, ret_bars
def get_bars(self,
symbol,
period,
callback,
bar_is_completed=False,
bar_freq=1,
start_dt=None):
"""
返回k线数据
symbol：股票 000001.XG
period: 周期: 1min,5min,15min,30min,1hour,1day,
"""
if self.api is None:
self.connect()
def __select_market_code(code):
code = str(code)
if code[0] in ['5', '6', '9'] or code[:3] in [
"009", "126", "110", "201", "202", "203", "204"
]:
return 1
return 0
# 新版一劳永逸偷懒写法zzz
if '.' in symbol:
tdx_code, market_str = symbol.split('.')
market_code = 1 if market_str.upper() == 'XSHG' else 0
self.symbol_exchange_dict.update({tdx_code:
symbol}) # tdx合约与vn交易所的字典
self.symbol_market_dict.update({tdx_code:
market_code}) # tdx合约与tdx市场的字典
else:
market_code = __select_market_code(symbol)
tdx_code = symbol
self.symbol_exchange_dict.update({symbol:
symbol}) # tdx合约与vn交易所的字典
self.symbol_market_dict.update({symbol:
market_code}) # tdx合约与tdx市场的字典
# https://github.com/rainx/pytdx/issues/33
# 0 - 深圳， 1 - 上海
ret_bars = []
if period not in PERIOD_MAPPING.keys():
self.strategy.writeCtaError(u'{} 周期{}不在下载清单中: {}'.format(
datetime.now(), period, list(PERIOD_MAPPING.keys())))
# print(u'{} 周期{}不在下载清单中: {}'.format(datetime.now(), period, list(PERIOD_MAPPING.keys())))
return False, ret_bars
if self.api is None:
return False, ret_bars
tdx_period = PERIOD_MAPPING.get(period)
if start_dt is None:
self.strategy.writeCtaLog(u'没有设置开始时间，缺省为10天前')
qry_start_date = datetime.now() - timedelta(days=10)
start_dt = qry_start_date
else:
qry_start_date = start_dt
end_date = datetime.now()
if qry_start_date > end_date:
qry_start_date = end_date
self.strategy.writeCtaLog('{}开始下载tdx股票:{} {}数据, {} to {}.'.format(
datetime.now(), tdx_code, tdx_period, qry_start_date, end_date))
try:
_start_date = end_date
_bars = []
_pos = 0
while _start_date > qry_start_date:
_res = self.api.get_security_bars(
category=PERIOD_MAPPING[period],
market=market_code,
code=tdx_code,
start=_pos,
count=QSIZE)
if _res is not None:
_bars = _res + _bars
_pos += QSIZE
if _res is not None and len(_res) > 0:
_start_date = _res[0]['datetime']
_start_date = datetime.strptime(_start_date,
'%Y-%m-%d %H:%M')
self.strategy.writeCtaLog(
u'分段取数据开始:{}'.format(_start_date))
else:
break
if len(_bars) == 0:
self.strategy.writeCtaError(
'{} Handling {}, len1={}..., continue'.format(
str(datetime.now()), tdx_code, len(_bars)))
return False, ret_bars
current_datetime = datetime.now()
data = self.api.to_df(_bars)
data = data.assign(datetime=pd.to_datetime(data['datetime']))
data = data.assign(ticker=symbol)
data['symbol'] = symbol
data = data.drop(
['year', 'month', 'day', 'hour', 'minute', 'price', 'ticker'],
errors='ignore',
axis=1)
data = data.rename(index=str, columns={
'amount': 'volume',
})
if len(data) == 0:
print('{} Handling {}, len2={}..., continue'.format(
str(datetime.now()), tdx_code, len(data)))
return False, ret_bars
# 通达信是以bar的结束时间标记的，vnpy是以bar开始时间标记的,所以要扣减bar本身的分钟数
data['datetime'] = data['datetime'].apply(lambda x: x - timedelta(
minutes=NUM_MINUTE_MAPPING.get(period, 1)))
data['trading_date'] = data['datetime'].apply(
lambda x: (x.strftime('%Y-%m-%d')))
data['date'] = data['datetime'].apply(lambda x:
(x.strftime('%Y-%m-%d')))
data['time'] = data['datetime'].apply(lambda x:
(x.strftime('%H:%M:%S')))
data = data.set_index('datetime', drop=False)
for index, row in data.iterrows():
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = index
add_bar.date = row['date']
add_bar.time = row['time']
add_bar.tradingDay = row['trading_date']
add_bar.open = float(row['open'])
add_bar.high = float(row['high'])
add_bar.low = float(row['low'])
add_bar.close = float(row['close'])
add_bar.volume = float(row['volume'])
except Exception as ex:
self.strategy.writeCtaError(
'error when convert bar:{},ex:{},t:{}'.format(
row, str(ex), traceback.format_exc()))
# print('error when convert bar:{},ex:{},t:{}'.format(row, str(ex), traceback.format_exc()))
return False
if start_dt is not None and index < start_dt:
continue
ret_bars.append(add_bar)
if callback is not None:
freq = bar_freq
bar_is_completed = True
if period != '1min' and index == data['datetime'][-1]:
# 最后一个bar，可能是不完整的，强制修改
# - 5min修改后freq基本正确
# - 1day在VNPY合成时不关心已经收到多少Bar, 所以影响也不大
# - 但其它分钟周期因为不好精确到每个品种, 修改后的freq可能有错
if index > current_datetime:
bar_is_completed = False
# 根据秒数算的话，要+1，例如13:31,freq=31，第31根bar
freq = NUM_MINUTE_MAPPING[period] - int(
(index - current_datetime).total_seconds() /
60)
callback(add_bar, bar_is_completed, freq)
return True, ret_bars
except Exception as ex:
self.strategy.writeCtaError('exception in get:{},{},{}'.format(
tdx_code, str(ex), traceback.format_exc()))
# print('exception in get:{},{},{}'.format(tdx_symbol,str(ex), traceback.format_exc()))
self.strategy.writeCtaLog(u'重置连接')
TdxData.api = None
self.connect()
return False, ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
tdx_symbol = symbol.upper().replace('_' , '')
tdx_symbol = tdx_symbol.replace('99' , 'L9')
if tdx_symbol not in self.symbol_exchange_dict.keys():
self.strategy.writeCtaError(u'{} 合约{}/{}不在下载清单中: {}'.format(datetime.now(), symbol, tdx_symbol, self.symbol_exchange_dict.keys()))
return False,ret_bars
if period not in PERIOD_MAPPING.keys():
self.strategy.writeCtaError(u'{} 周期{}不在下载清单中: {}'.format(datetime.now(), period, list(PERIOD_MAPPING.keys())))
return False,ret_bars
if self.api is None:
return False,ret_bars
tdx_period = PERIOD_MAPPING.get(period)
if start_dt is None:
self.strategy.writeCtaLog(u'没有设置开始时间，缺省为10天前')
qry_start_date = datetime.now() - timedelta(days=10)
else:
qry_start_date = start_dt
end_date = datetime.combine(datetime.now() + timedelta(days=1),time(ALL_MARKET_END_HOUR, 0))
if qry_start_date > end_date:
qry_start_date = end_date
self.strategy.writeCtaLog('{}开始下载tdx:{} {}数据, {} to {}.'.format(datetime.now(), tdx_symbol, period, qry_start_date, end_date))
# print('{}开始下载tdx:{} {}数据, {} to {}.'.format(datetime.now(), tdx_symbol, tdx_period, last_date, end_date))
try:
_start_date = end_date
_bars = []
_pos = 0
while _start_date > qry_start_date:
_res = self.api.get_instrument_bars(
PERIOD_MAPPING[period],
self.symbol_market_dict.get(tdx_symbol,0),
tdx_symbol,
_pos,
QSIZE)
if _res is not None:
_bars = _res + _bars
_pos += QSIZE
if _res is not None and len(_res) > 0:
_start_date = _res[0]['datetime']
_start_date = datetime.strptime(_start_date, '%Y-%m-%d %H:%M')
self.strategy.writeCtaLog(u'分段取数据开始:{}'.format(_start_date))
else:
break
if len(_bars) == 0:
self.strategy.writeCtaError('{} Handling {}, len1={}..., continue'.format(
str(datetime.now()), tdx_symbol, len(_bars)))
return False, ret_bars
current_datetime = datetime.now()
data = self.api.to_df(_bars)
data = data.assign(datetime=pd.to_datetime(data['datetime']))
data = data.assign(ticker=symbol)
data['instrument_id'] = data['ticker']
# if future['market'] == 28 or future['market'] == 47:
# # 大写字母: 郑州商品 or 中金所期货
# data['instrument_id'] = data['ticker']
# else:
# data['instrument_id'] = data['ticker'].apply(lambda x: x.lower())
data['symbol'] = symbol
data = data.drop(
['year', 'month', 'day', 'hour', 'minute', 'price', 'amount', 'ticker'],
errors='ignore',
axis=1)
data = data.rename(
index=str,
columns={
'position': 'open_interest',
'trade': 'volume',
})
if len(data) == 0:
print('{} Handling {}, len2={}..., continue'.format(
str(datetime.now()), tdx_symbol, len(data)))
return False, ret_bars
data['total_turnover'] = data['volume']
data["limit_down"] = 0
data["limit_up"] = 999999
data['trading_date'] = data['datetime']
data['trading_date'] = data['trading_date'].apply(lambda x: (x.strftime('%Y-%m-%d')))
monday_ts = data['datetime'].dt.weekday == 0 # 星期一
night_ts1 = data['datetime'].dt.hour > ALL_MARKET_END_HOUR
night_ts2 = data['datetime'].dt.hour < ALL_MARKET_BEGIN_HOUR
data.loc[night_ts1, 'datetime'] -= timedelta(days=1) # 所有日期的夜盘(21:00~24:00), 减一天
monday_ts1 = monday_ts & night_ts1 # 星期一的夜盘(21:00~24:00), 再减两天
data.loc[monday_ts1, 'datetime'] -= timedelta(days=2)
monday_ts2 = monday_ts & night_ts2 # 星期一的夜盘(00:00~04:00), 再减两天
data.loc[monday_ts2, 'datetime'] -= timedelta(days=2)
# data['datetime'] -= timedelta(minutes=1) # 直接给Strategy使用, RiceQuant格式, 不需要减1分钟
data['dt_datetime'] = data['datetime']
data['date'] = data['datetime'].apply(lambda x: (x.strftime('%Y-%m-%d')))
data['time'] = data['datetime'].apply(lambda x: (x.strftime('%H:%M:%S')))
data['datetime'] = data['datetime'].apply(lambda x: float(x.strftime('%Y%m%d%H%M%S')))
data = data.set_index('dt_datetime', drop=False)
# data = data[int(last_date.strftime('%Y%m%d%H%M%S')):int(end_date.strftime('%Y%m%d%H%M%S'))]
# data = data[str(last_date):str(end_date)]
for index, row in data.iterrows():
add_bar = CtaBarData()
try:
add_bar.vtSymbol = row['symbol']
add_bar.symbol = row['symbol']
add_bar.datetime = index
add_bar.date = row['date']
add_bar.time = row['time']
add_bar.tradingDay = row['trading_date']
add_bar.open = float(row['open'])
add_bar.high = float(row['high'])
add_bar.low = float(row['low'])
add_bar.close = float(row['close'])
add_bar.volume = float(row['volume'])
add_bar.openInterest = float(row['open_interest'])
except Exception as ex:
self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(row, str(ex), traceback.format_exc()))
# print('error when convert bar:{},ex:{},t:{}'.format(row, str(ex), traceback.format_exc()))
return False
if start_dt is not None and index < start_dt:
continue
ret_bars.append(add_bar)
if callback is not None:
freq = bar_freq
bar_is_completed = True
if period != '1min' and index == data['dt_datetime'][-1]:
# 最后一个bar，可能是不完整的，强制修改
# - 5min修改后freq基本正确
# - 1day在VNPY合成时不关心已经收到多少Bar, 所以影响也不大
# - 但其它分钟周期因为不好精确到每个品种, 修改后的freq可能有错
if index > current_datetime:
bar_is_completed = False
# 根据秒数算的话，要+1，例如13:31,freq=31，第31根bar
freq = NUM_MINUTE_MAPPING[period] - int((index - current_datetime).total_seconds() / 60)
callback(add_bar, bar_is_completed, freq)
return True, ret_bars
except Exception as ex:
self.strategy.writeCtaError('exception in get:{},{},{}'.format(tdx_symbol,str(ex), traceback.format_exc()))
# print('exception in get:{},{},{}'.format(tdx_symbol,str(ex), traceback.format_exc()))
self.strategy.writeCtaLog(u'重置连接')
TdxFutureData.api = None
self.connect()
return False,ret_bars
def get_bars(self,
symbol,
period,
callback,
bar_is_completed=False,
bar_freq=1,
start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
if period not in period_list:
self.strategy.writeCtaError('{} not in {}'.format(
period, period_list))
return False, ret_bars
period = period.replace('min', 'm').replace('hour',
'h').replace('day', 'd')
count = 750
if start_dt is None:
if period == '1d':
start_dt = datetime.utcnow() - timedelta(days=count - 1)
elif period == '1h':
print(u'utc now():{}'.format(datetime.utcnow()))
start_dt = datetime.utcnow() - timedelta(hours=count - 1)
print(u'start_dt():{}'.format(start_dt))
elif period == '5m':
start_dt = datetime.utcnow() - timedelta(minutes=(count - 1) *
5)
else:
start_dt = datetime.utcnow() - timedelta(minutes=count - 1)
path = '/trade/bucketed'
url = u'https://www.bitmex.com/api/v1' + path
params = {
'binSize': period,
'partial': True,
'symbol': symbol,
'startTime': start_dt.strftime('%Y-%m-%d %H:%M'),
'count': count
}
params = OrderedDict(sorted(params.items()))
expires = int(time() + 5)
rq = requests.Request(url=url)
p = rq.prepare()
header = {
'Content-Type':
'application/x-www-form-urlencoded',
'Accept':
'application/json',
'api-expires':
str(expires),
'api-key':
self.apiKey,
'api-signature':
self.generateSignature('GET', path, expires, params, body=p.body)
}
print(header)
session = requests.Session()
resp = session.request('GET',
url,
headers=header,
params=params,
data=None)
self.writeLog('{}开始下载:{} {}数据.开始时间:{} URL:{}'.format(
datetime.now(), symbol, period, start_dt, url))
bars = None
code = resp.status_code
bars = resp.json()
if code != 200:
self.strategy.writeCtaError('get_bars {},{}'.format(code, bars))
return False, ret_bars
bar_len = len(bars)
self.writeLog(u'共下载:{}条'.format(bar_len))
for i, bar in enumerate(bars):
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
utc_str_time = bar.get('timestamp').replace('T', ' ').replace(
'Z', '')
utc_datetime = datetime.strptime(utc_str_time,
'%Y-%m-%d %H:%M:%S.%f')
add_bar.datetime = utc_datetime + timedelta(hours=8)
# 时区得调整 +8
if period == '1h':
add_bar.datetime = add_bar.datetime - timedelta(hours=1)
elif period == '5m':
add_bar.datetime = add_bar.datetime - timedelta(minutes=5)
elif period == '1m':
add_bar.datetime = add_bar.datetime - timedelta(minutes=1)
# bitmex是以bar得结束时间，作为bar得datetime，vnpy是以bar得开始时间作为datetime
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.open = float(bar.get('open', 0.0))
add_bar.high = float(bar.get('high', 0.0))
add_bar.low = float(bar.get('low', 0.0))
add_bar.close = float(bar.get('close', 0.0))
add_bar.volume = float(bar.get('volume', 0.0))
ret_bars.append(add_bar)
except Exception as ex:
self.strategy.writeCtaError(
'error when convert bar:{},ex:{},t:{}'.format(
bar, str(ex), traceback.format_exc()))
return False, ret_bars
if start_dt is not None and add_bar.datetime < start_dt:
continue
if callback is not None:
# 最后一个bar，可能是不完整的，强制修改
if i == bar_len - 1 and bar_is_completed:
# 根据秒数算的话，要+1，例如13:31,freq=31，第31根bar
freq = int(
(datetime.now() - add_bar.datetime).total_seconds() /
60) + 1
callback(add_bar, False, freq)
else:
callback(add_bar, bar_is_completed, bar_freq)
return True, ret_bars
def get_bars(self,
symbol,
period,
callback,
bar_is_completed=False,
bar_freq=1,
start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
if symbol not in symbol_list:
self.strategy.writeCtaError(u'{} {}不在下载清单中'.format(
datetime.now(), symbol))
return False, ret_bars
group_sec, range_hour = self.get_sec_hours(period)
url = u'https://data.gateio.io/api2/1/candlestick2/{}?group_sec={}&range_hour={}'.format(
symbol, group_sec, range_hour)
self.strategy.writeCtaLog('{}开始下载:{} {}数据.URL:{}'.format(
datetime.now(), symbol, period, url))
content = None
try:
content = self.session.get(url).content.decode('gbk')
except Exception as ex:
self.strategy.writeCtaError('exception in get:{},{},{}'.format(
url, str(ex), traceback.format_exc()))
return False, ret_bars
content = execjs.eval(content)
if not isinstance(content, dict):
self.strategy.writeCtaError(
u'content is not dict format:{}'.format(content))
return False, ret_bars
bars = content.get('data', [])
for i, bar in enumerate(bars):
if len(bar) < 6:
self.strategy.writeCtaError(
'error when import bar:{}'.format(bar))
return False, ret_bars
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.fromtimestamp(int(bar[0]) / 1000)
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.volume = float(bar[1])
add_bar.close = float(bar[2])
add_bar.high = float(bar[3])
add_bar.low = float(bar[4])
add_bar.open = float(bar[5])
except Exception as ex:
self.strategy.writeCtaError(
'error when convert bar:{},ex:{},t:{}'.format(
bar, str(ex), traceback.format_exc()))
return False, ret_bars
if start_dt is not None and bar.datetime < start_dt:
continue
ret_bars.append(add_bar)
if callback is not None:
callback(add_bar, bar_is_completed, bar_freq)
return True, ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False,bar_freq=1, start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
if symbol not in symbol_list:
self.strategy.writeCtaError(u'{} {}不在下载清单中'.format(datetime.now(), symbol))
return
url = u'https://www.okex.com/api/v1/kline.do?symbol={}&type={}'.format(symbol, period)
self.strategy.writeCtaLog('{}开始下载:{} {}数据.URL:{}'.format(datetime.now(), symbol, period,url))
content = None
try:
content = self.session.get(url).content.decode('gbk')
except Exception as ex:
self.strategy.writeCtaError('exception in get:{},{},{}'.format(url,str(ex), traceback.format_exc()))
return
bars = execjs.eval(content)
for i, bar in enumerate(bars):
if len(bar) < 5:
self.strategy.writeCtaError('error when import bar:{}'.format(bar))
return False
if i == 0:
continue
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000)
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.open = float(bar[1])
add_bar.high = float(bar[2])
add_bar.low = float(bar[3])
add_bar.close = float(bar[4])
add_bar.volume = float(bar[5])
except Exception as ex:
self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc()))
return False
if start_dt is not None and bar.datetime < start_dt:
continue
if callback is not None:
callback(add_bar, bar_is_completed, bar_freq)
return True
def getMinBars(self, symbol, minute, callback,start_dt=None):
"""
从sina加载最新的M5,M15,M30,M60数据
:param symbol: （全路径得合约名称,先使用ctaTemplate.getFullSymbol()
:param minute: 5，15，30，60
:param callback: 回调函数
:param start_dt: 开始时间，缺省为None
:return: 成功/失败
"""
if minute not in [5, 15, 30, 60]:
return False
sinaBars = []
try:
url = u'http://stock2.finance.sina.com.cn/futures/api/json.php/InnerFuturesService.getInnerFutures{0}MinKLine?symbol={1}'.format(minute,symbol)
self.strategy.writeCtaLog(u'从sina下载{0}的{1}分钟数据 {2}'.format(symbol,minute, url))
responses = execjs.eval(self.session.get(url).content.decode('gbk').split('\n')[-1])
dayVolume = 0
for item in responses:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
# bar的close time
sinaDt = datetime.strptime(item[0], '%Y-%m-%d %H:%M:00')
if minute in {5, 15} and sinaDt.hour == 10 and sinaDt.minute == 30:
# 这个是sina的bug，它把10:15 ~10:30也包含进来了
continue
if minute == 60 and sinaDt.hour in {11,23,1,2} and sinaDt.minute == 30:
bar.datetime = sinaDt - timedelta(seconds=(minute /2)* 60)
else:
bar.datetime = sinaDt - timedelta(seconds=minute * 60)
if start_dt is not None:
if bar.datetime < start_dt:
continue
bar.date = bar.datetime.strftime('%Y%m%d')
bar.tradingDay = bar.date # todo: 需要修改，晚上21点后，修改为next workingday
bar.time = bar.datetime.strftime('%H:%M:00')
bar.open = float(item[1])
bar.high = float(item[2])
bar.low = float(item[3])
bar.close = float(item[4])
bar.volume = int(item[5])
# 计算dayvolume
if not sinaBars:
dayVolume = bar.volume
else:
if sinaBars[-1].datetime.hour == 14 and bar.datetime.hour !=14:
dayVolume = bar.volume
else:
dayVolume += bar.volume
bar.dayVolume = dayVolume
sinaBars.append(bar)
if len(sinaBars)>0:
self.strategy.writeCtaLog(u'从sina读取了{0}条{1}分钟数据'.format(len(sinaBars),minute))
# 把sina的bar灌入回调函数
for bar in sinaBars:
callback(bar)
# 处理完毕，清空
sinaBars = []
return True
else:
self.strategy.writeCtaLog(u'从sina读取{0}分钟数据失败'.format(minute))
return False
except Exception as e:
self.strategy.writeCtaLog(u'加载Sina历史分钟数据失败：'+str(e))
return False
def getMinBars2(self, symbol, minute, callback,start_dt=None):
"""
从sina加载最新的M5,M15,M30,M60数据(针对中金所）
:param symbol: （全路径得合约名称,先使用ctaTemplate.getFullSymbol()
:param minute: 5，15，30，60
:param callback: 回调函数
:param start_dt: 开始时间，缺省为None
:return: 成功/失败
"""
if minute not in {5, 15, 30, 60}:
return False
sinaBars = []
try:
timestamp = (datetime.utcnow() - datetime(1970, 1, 1)).total_seconds()
url=u'http://stock2.finance.sina.com.cn/futures/api/jsonp.php/var%20_{1}_{0}_{2}=/InnerFuturesNewService.getFewMinLine?symbol={1}&type={0}'.format(minute,symbol,timestamp)
#url = u'http://stock2.finance.sina.com.cn/futures/api/json.php/InnerFuturesService.getInnerFutures{0}MinKLine?symbol={1}'.format(minute,symbol)
self.strategy.writeCtaLog(u'从sina下载{0}的{1}分钟数据 {2}'.format(symbol,minute, url))
response_data = self.session.get(url).content
response_data = response_data.decode('gbk').split('=')[-1]
response_data = response_data.replace('(', '')
response_data = response_data.replace(');', '')
responses = execjs.eval(response_data)
dayVolume = 0
for item in responses:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
# bar的close time
sinaDt = datetime.strptime(item['d'], '%Y-%m-%d %H:%M:00')
if minute in {5, 15} and sinaDt.hour == 10 and sinaDt.minute == 30:
# 这个是sina的bug，它把10:15 ~10:30也包含进来了
continue
if minute == 60 and sinaDt.hour in {11,23,1,2} and sinaDt.minute == 30:
bar.datetime = sinaDt - timedelta(seconds=(minute /2)* 60)
else:
bar.datetime = sinaDt - timedelta(seconds=minute * 60)
if start_dt is not None:
if bar.datetime < start_dt:
continue
#bar.tradingDay = bar.date # todo: 需要修改，晚上21点后，修改为next workingday
if bar.datetime.hour >= 21:
if bar.datetime.isoweekday() == 5:
# 星期五=》星期一
bar.tradingDay = (bar.datetime + timedelta(days=3)).strftime('%Y-%m-%d')
else:
# 第二天
bar.tradingDay = (bar.datetime + timedelta(days=1)).strftime('%Y-%m-%d')
elif bar.datetime.hour < 8 and bar.datetime.isoweekday() == 6:
# 星期六=>星期一
bar.tradingDay = (bar.datetime + timedelta(days=2)).strftime('%Y-%m-%d')
else:
bar.tradingDay = bar.date
bar.time = bar.datetime.strftime('%H:%M:00')
bar.open = float(item['o'])
bar.high = float(item['h'])
bar.low = float(item['l'])
bar.close = float(item['c'])
bar.volume = int(item['v'])
# 计算dayvolume
if not sinaBars:
dayVolume = bar.volume
else:
if sinaBars[-1].datetime.hour == 14 and bar.datetime.hour !=14:
dayVolume = bar.volume
else:
dayVolume += bar.volume
bar.dayVolume = dayVolume
sinaBars.append(bar)
if len(sinaBars)>0:
self.strategy.writeCtaLog(u'从sina读取了{0}条{1}分钟数据'.format(len(sinaBars),minute))
# 把sina的bar灌入回调函数
for bar in sinaBars:
callback(bar)
# 处理完毕，清空
sinaBars = []
return True
else:
self.strategy.writeCtaLog(u'从sina读取{0}分钟数据失败'.format(minute))
return False
except Exception as e:
self.strategy.writeCtaLog(u'加载Sina历史分钟数据失败：'+str(e))
return False
def get_bars(self,
symbol,
period,
callback,
bar_is_completed=False,
bar_freq=1,
start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
symbol = symbol.upper()
if symbol not in symbol_list:
msg = u'{} {}不在下载清单中'.format(datetime.now(), symbol)
if self.strategy:
self.strategy.writeCtaError(msg)
else:
print(msg)
return False, ret_bars
if period not in period_list:
self.writeError(u'{}不在下载时间周期范围:{} 内'.format(period, period_list))
return False, ret_bars
candleCount = 200
url = u'https://www.bitmex.com/api/v1/trade/bucketed?binSize={}&partial=false&symbol={}&count={}&reverse=true'.format(
period, symbol, candleCount)
self.writeLog('{}开始下载:{} {}数据.URL:{}'.format(datetime.now(), symbol,
period, url))
content = None
try:
content = self.session.get(url).content.decode('gbk')
except Exception as ex:
self.writeError('exception in get:{},{},{}'.format(
url, str(ex), traceback.format_exc()))
return False, ret_bars
bars = execjs.eval(content)
if not isinstance(bars, list):
self.writeError('返回数据不是list:{}'.format(content))
return False, ret_bars
# example bar:
# {"timestamp":"2018-08-09T02:37:00.000Z","symbol":"XBTUSD","open":6313.5,"high":6313.5,"low":6313,"close":6313.5,
# "trades":63,"volume":243870,"vwap":6313.5299,"lastSize":1000,"turnover":3862773286,"homeNotional":38.62773286,
# "foreignNotional":243870}
for i, bar in enumerate(bars):
add_bar = CtaBarData()
try:
add_bar.vtSymbol = bar['symbol']
add_bar.symbol = bar['symbol']
raw_datetime = bar['timestamp']
raw_datetime = raw_datetime.split('T')
add_bar.date = raw_datetime[0]
add_bar.time = raw_datetime[1][:-5]
date = add_bar.date.split('-')
time = add_bar.time.split(':')
add_bar.datetime = datetime(int(date[0]), int(date[1]),
int(date[2]), int(time[0]),
int(time[1]), int(time[2]))
add_bar.tradingDay = add_bar.date
add_bar.open = bar['open']
add_bar.high = bar['high']
add_bar.low = bar['low']
add_bar.close = bar['close']
add_bar.volume = bar['volume']
except Exception as ex:
self.strategy.writeCtaError(
'error when convert bar:{},ex:{},t:{}'.format(
bar, str(ex), traceback.format_exc()))
return False, ret_bars
if start_dt is not None and bar.datetime < start_dt:
continue
if callback is not None:
callback(add_bar, bar_is_completed, bar_freq)
ret_bars.append(add_bar)
return True, ret_bars
def getDayBars(self, symbol, callback,start_dt=None):
"""
从sina加载最新的Day数据
:param symbol: （全路径得合约名称,先使用ctaTemplate.getFullSymbol()
:param callback: 回调函数
:param start_dt: 开始时间，缺省为None
:return: 成功/失败
"""
sinaBars = []
try:
url = u'http://stock.finance.sina.com.cn/futures/api/json.php/InnerFuturesService.getInnerFuturesDailyKLine?symbol={0}'.format(symbol)
self.strategy.writeCtaLog(u'从sina下载{0}的日K数据 {1}'.format(symbol, url))
responses = execjs.eval(self.session.get(url).content.decode('gbk'))
dayVolume = 0
for item in responses:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
# bar的close time
bar.datetime = datetime.strptime(item['date'], '%Y-%m-%d')
if start_dt is not None:
if bar.datetime < start_dt:
continue
bar.date = bar.datetime.strftime('%Y%m%d')
bar.tradingDay = bar.date
bar.time = bar.datetime.strftime('%H:%M:00')
bar.open = float(item['open'])
bar.high = float(item['high'])
bar.low = float(item['low'])
bar.close = float(item['close'])
bar.volume = int(item['volume'])
bar.dayVolume = bar.volume
sinaBars.append(bar)
if len(sinaBars)>0:
self.strategy.writeCtaLog(u'从sina读取了{0}条日线K数据'.format(len(sinaBars)))
# 把sina的bar灌入回调函数
for bar in sinaBars:
callback(bar)
# 处理完毕，清空
sinaBars = []
return True
else:
self.strategy.writeCtaLog(u'从sina读取日线K数据失败')
return False
except Exception as e:
self.strategy.writeCtaLog(u'加载Sina历史日线数据失败：'+str(e))
return False
def getMinBars2(self, symbol, minute, callback):
"""# 从sina加载最新的M5,M15,M30,M60数据(针对中金所）"""
if minute not in {5, 15, 30, 60}:
return False
sinaBars = []
try:
timestamp = (datetime.utcnow() -
datetime(1970, 1, 1)).total_seconds()
url = u'http://stock2.finance.sina.com.cn/futures/api/jsonp.php/var%20_{1}_{0}_{2}=/InnerFuturesNewService.getFewMinLine?symbol={1}&type={0}'.format(
minute, symbol, timestamp)
#url = u'http://stock2.finance.sina.com.cn/futures/api/json.php/InnerFuturesService.getInnerFutures{0}MinKLine?symbol={1}'.format(minute,symbol)
self.strategy.writeCtaLog(u'从sina下载{0}的{1}分钟数据 {2}'.format(
symbol, minute, url))
response_data = self.session.get(url).content
response_data = response_data.decode('gbk').split('=')[-1]
response_data = response_data.replace('(', '')
response_data = response_data.replace(');', '')
responses = execjs.eval(response_data)
dayVolume = 0
for item in responses:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
# bar的close time
sinaDt = datetime.strptime(item['d'], '%Y-%m-%d %H:%M:00')
if minute in {5, 15
} and sinaDt.hour == 10 and sinaDt.minute == 30:
# 这个是sina的bug，它把10:15 ~10:30也包含进来了
continue
if minute == 60 and sinaDt.hour in {11, 23, 1, 2
} and sinaDt.minute == 30:
bar.datetime = sinaDt - timedelta(seconds=(minute / 2) *
60)
else:
bar.datetime = sinaDt - timedelta(seconds=minute * 60)
bar.date = bar.datetime.strftime('%Y%m%d')
bar.tradingDay = bar.date # todo: 需要修改，晚上21点后，修改为next workingday
bar.time = bar.datetime.strftime('%H:%M:00')
bar.open = float(item['o'])
bar.high = float(item['h'])
bar.low = float(item['l'])
bar.close = float(item['c'])
bar.volume = int(item['v'])
# 计算dayvolume
if not sinaBars:
dayVolume = bar.volume
else:
if sinaBars[
-1].datetime.hour == 14 and bar.datetime.hour != 14:
dayVolume = bar.volume
else:
dayVolume += bar.volume
bar.dayVolume = dayVolume
sinaBars.append(bar)
if len(sinaBars) > 0:
self.strategy.writeCtaLog(u'从sina读取了{0}条{1}分钟数据'.format(
len(sinaBars), minute))
# 把sina的bar灌入回调函数
for bar in sinaBars:
callback(bar)
# 处理完毕，清空
sinaBars = []
return True
else:
self.strategy.writeCtaLog(u'从sina读取{0}分钟数据失败'.format(minute))
return False
except Exception as e:
self.strategy.writeCtaLog(u'加载Sina历史分钟数据失败：' + str(e))
return False
def get_bars(self, symbol, period, callback, bar_is_completed=False,bar_freq=1, start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
if symbol not in symbol_list:
self.strategy.writeCtaError(u'{} {}不在下载清单中'.format(datetime.now(), symbol))
return False,ret_bars
url = u'https://www.okex.com/api/v1/kline.do?symbol={}&type={}'.format(symbol, period)
self.writeLog('{}开始下载:{} {}数据.URL:{}'.format(datetime.now(), symbol, period,url))
content = None
try:
content = self.session.get(url).content.decode('gbk')
except Exception as ex:
self.strategy.writeCtaError('exception in get:{},{},{}'.format(url,str(ex), traceback.format_exc()))
return False,ret_bars
bars = execjs.eval(content)
bar_len = len(bars)
for i, bar in enumerate(bars):
if len(bar) < 5:
self.strategy.writeCtaError('error when import bar:{}'.format(bar))
return False,ret_bars
if i == 0:
continue
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000)
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.open = float(bar[1])
add_bar.high = float(bar[2])
add_bar.low = float(bar[3])
add_bar.close = float(bar[4])
add_bar.volume = float(bar[5])
ret_bars.append(add_bar)
except Exception as ex:
self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc()))
return False,ret_bars
if start_dt is not None and bar.datetime < start_dt:
continue
if callback is not None:
# 最后一个bar，可能是不完整的，强制修改
if i == bar_len -1 and bar_is_completed:
# 根据秒数算的话，要+1，例如13:31,freq=31，第31根bar
freq = int((datetime.now() - add_bar.datetime).total_seconds()/60)+1
callback(add_bar,False,freq)
else:
callback(add_bar, bar_is_completed, bar_freq)
return True,ret_bars
def getDayBars(self, symbol, callback):
"""# 从sina加载最新的Day数据"""
sinaBars = []
try:
url = u'http://stock.finance.sina.com.cn/futures/api/json.php/InnerFuturesService.getInnerFuturesDailyKLine?symbol={0}'.format(
symbol)
self.strategy.writeCtaLog(u'从sina下载{0}的日K数据 {1}'.format(
symbol, url))
responses = execjs.eval(
self.session.get(url).content.decode('gbk'))
dayVolume = 0
for item in responses:
bar = CtaBarData()
bar.vtSymbol = symbol
bar.symbol = symbol
# bar的close time
bar.datetime = datetime.strptime(item['date'], '%Y-%m-%d')
bar.date = bar.datetime.strftime('%Y%m%d')
bar.tradingDay = bar.date # todo: 需要修改，晚上21点后，修改为next workingday
bar.time = bar.datetime.strftime('%H:%M:00')
bar.open = float(item['open'])
bar.high = float(item['high'])
bar.low = float(item['low'])
bar.close = float(item['close'])
bar.volume = int(item['volume'])
bar.dayVolume = bar.volume
sinaBars.append(bar)
if len(sinaBars) > 0:
self.strategy.writeCtaLog(u'从sina读取了{0}条日线K数据'.format(
len(sinaBars)))
# 把sina的bar灌入回调函数
for bar in sinaBars:
callback(bar)
# 处理完毕，清空
sinaBars = []
return True
else:
self.strategy.writeCtaLog(u'从sina读取日线K数据失败')
return False
except Exception as e:
self.strategy.writeCtaLog(u'加载Sina历史日线数据失败：' + str(e))
return False
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None):
"""
返回k线数据
symbol：合约b tc:next_week:10
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
status, bars = self.get_data(symbol, period, start_dt)
if not status:
return False, bars
for i, bar in enumerate(bars):
if len(bar) < 5:
self.strategy.writeCtaError('error when import bar:{}'.format(bar))
return False
add_bar = CtaBarData()
try:
time = bar[0].replace('T', ' ')[:-5]
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.strptime(time, '%Y-%m-%d %H:%M:%S') + timedelta(hours=8)
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.open = float(bar[1])
add_bar.high = float(bar[2])
add_bar.low = float(bar[3])
add_bar.close = float(bar[4])
add_bar.volume = float(bar[6]) # 这里：5 是交易量，6是交易量转化BTC或LTC数量
except Exception as ex:
self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc()))
return False, ret_bars
ret_bars.append(add_bar)
if callback is not None:
callback(add_bar, bar_is_completed, bar_freq)
return True, ret_bars
def get_bars(self,
symbol,
period,
callback,
bar_is_completed=False,
bar_freq=1,
start_dt=None):
"""
返回k线数据
symbol：合约
period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour
"""
ret_bars = []
symbol_pair = systemSymbolToVnSymbol(symbol)
if symbol_pair not in symbol_list:
msg = u'{} {}不在下载清单中'.format(datetime.now(), symbol_pair)
if self.strategy:
self.strategy.writeCtaError(msg)
else:
print(msg)
return False, ret_bars
symbol = symbol_pair.replace('_', '')
if period == '1hour':
period = '60min'
if period not in period_list:
self.writeError(u'{}不在下载时间周期范围:{} 内'.format(period, period_list))
return False, ret_bars
url = u'https://api.huobipro.com/market/history/kline?symbol={}&period={}& size=2000&AccessKeyId=fff-xxx-ssss-kkk'.format(
symbol, period)
self.writeLog('{}开始下载:{} {}数据.URL:{}'.format(datetime.now(),
symbol_pair, period, url))
content = None
try:
content = self.session.get(url).content.decode('gbk')
except Exception as ex:
self.writeError('exception in get:{},{},{}'.format(
url, str(ex), traceback.format_exc()))
return False, ret_bars
ret_dict = execjs.eval(content)
bars = ret_dict.get('data', None)
if not isinstance(bars, list):
self.writeError('返回数据不是list:{}'.format(content))
return False, ret_bars
bars.reverse()
for i, bar in enumerate(bars):
add_bar = CtaBarData()
try:
add_bar.vtSymbol = symbol
add_bar.symbol = symbol
add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000)
add_bar.date = add_bar.datetime.strftime('%Y-%m-%d')
add_bar.time = add_bar.datetime.strftime('%H:%M:%S')
add_bar.tradingDay = add_bar.date
add_bar.open = float(bar[1])
add_bar.high = float(bar[2])
add_bar.low = float(bar[3])
add_bar.close = float(bar[4])
add_bar.volume = float(bar[5])
except Exception as ex:
self.strategy.writeCtaError(
'error when convert bar:{},ex:{},t:{}'.format(
bar, str(ex), traceback.format_exc()))
return False, ret_bars
if start_dt is not None and bar.datetime < start_dt:
continue
if callback is not None:
callback(add_bar, bar_is_completed, bar_freq)
return True, ret_bars