def updateTick(self, tick): """TICK更新""" newMinute = False # 默认不是新的一分钟 # 尚未创建对象 if not self.bar: #self.bar = VtBarData() self.bar = CtaBarData() newMinute = True # 新的一分钟 elif self.bar.datetime.minute != tick.datetime.minute: # 生成上一分钟K线的时间戳 self.bar.datetime = self.bar.datetime.replace( second=0, microsecond=0) # 将秒和微秒设为0 self.bar.date = self.bar.datetime.strftime('%Y-%m-%d') self.bar.time = self.bar.datetime.strftime('%H:%M:%S') self.bar.tradingDay = self.bar.date # 推送已经结束的上一分钟K线 self.onBar(self.bar) print(u'BarGenerate:Onbar:{},{},o:{},h:{},l:{},c:{}'.format( self.bar.vtSymbol, self.bar.datetime, self.bar.open, self.bar.high, self.bar.low, self.bar.close)) # 创建新的K线对象 #self.bar = VtBarData() self.bar = CtaBarData() newMinute = True # 初始化新一分钟的K线数据 if newMinute: self.bar.vtSymbol = tick.vtSymbol self.bar.symbol = tick.symbol self.bar.exchange = tick.exchange self.bar.open = tick.lastPrice self.bar.high = tick.lastPrice self.bar.low = tick.lastPrice # 累加更新老一分钟的K线数据 else: self.bar.high = max(self.bar.high, tick.lastPrice) self.bar.low = min(self.bar.low, tick.lastPrice) # 通用更新部分 self.bar.close = tick.lastPrice self.bar.datetime = tick.datetime self.bar.openInterest = tick.openInterest if self.lastTick: volumeChange = tick.volume - self.lastTick.volume # 当前K线内的成交量 self.bar.volume += max( volumeChange, 0) # 避免夜盘开盘lastTick.volume为昨日收盘数据,导致成交量变化为负的情况 # 缓存Tick self.lastTick = tick
def get_bars(self, symbol, period, callback, bar_is_completed=False,bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] if symbol not in symbol_list: self.strategy.writeCtaError(u'{} {}不在下载清单中'.format(datetime.now(), symbol)) return False,ret_bars url = u'https://www.okex.com/api/v1/kline.do?symbol={}&type={}'.format(symbol, period) self.writeLog('{}开始下载:{} {}数据.URL:{}'.format(datetime.now(), symbol, period,url)) content = None try: content = self.session.get(url).content.decode('gbk') except Exception as ex: self.strategy.writeCtaError('exception in get:{},{},{}'.format(url,str(ex), traceback.format_exc())) return False,ret_bars bars = execjs.eval(content) bar_len = len(bars) for i, bar in enumerate(bars): if len(bar) < 5: self.strategy.writeCtaError('error when import bar:{}'.format(bar)) return False,ret_bars if i == 0: continue add_bar = CtaBarData() try: add_bar.vtSymbol = symbol add_bar.symbol = symbol add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000) add_bar.date = add_bar.datetime.strftime('%Y-%m-%d') add_bar.time = add_bar.datetime.strftime('%H:%M:%S') add_bar.tradingDay = add_bar.date add_bar.open = float(bar[1]) add_bar.high = float(bar[2]) add_bar.low = float(bar[3]) add_bar.close = float(bar[4]) add_bar.volume = float(bar[5]) ret_bars.append(add_bar) except Exception as ex: self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc())) return False,ret_bars if start_dt is not None and bar.datetime < start_dt: continue if callback is not None: # 最后一个bar,可能是不完整的,强制修改 if i == bar_len -1 and bar_is_completed: # 根据秒数算的话,要+1,例如13:31,freq=31,第31根bar freq = int((datetime.now() - add_bar.datetime).total_seconds()/60)+1 callback(add_bar,False,freq) else: callback(add_bar, bar_is_completed, bar_freq) return True,ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约b tc:next_week:10 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] status, bars = self.get_data(symbol, period, start_dt) if not status: return False, bars for i, bar in enumerate(bars): if len(bar) < 5: self.strategy.writeCtaError('error when import bar:{}'.format(bar)) return False add_bar = CtaBarData() try: time = bar[0].replace('T', ' ')[:-5] add_bar.vtSymbol = symbol add_bar.symbol = symbol add_bar.datetime = datetime.strptime(time, '%Y-%m-%d %H:%M:%S') + timedelta(hours=8) add_bar.date = add_bar.datetime.strftime('%Y-%m-%d') add_bar.time = add_bar.datetime.strftime('%H:%M:%S') add_bar.tradingDay = add_bar.date add_bar.open = float(bar[1]) add_bar.high = float(bar[2]) add_bar.low = float(bar[3]) add_bar.close = float(bar[4]) add_bar.volume = float(bar[6]) # 这里:5 是交易量,6是交易量转化BTC或LTC数量 except Exception as ex: self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc())) return False, ret_bars ret_bars.append(add_bar) if callback is not None: callback(add_bar, bar_is_completed, bar_freq) return True, ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False,bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约b tc:next_week:10 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] if ':' not in symbol: self.strategy.writeCtaError(u'{} {}格式需要包含合约类型,如:btc:next_week:10'.format(datetime.now(), symbol)) return False, ret_bars s = symbol.split(':') symbol_pair, contract_type = s[0],s[1] if not symbol_pair.endswith('_usd'): symbol_pair += '_usd' if symbol_pair not in symbol_list: self.strategy.writeCtaError(u'{} {}不在下载清单中'.format(datetime.now(), symbol_pair)) return False, ret_bars url = u'https://www.okex.com/api/v1/future_kline.do?symbol={}&type={}&contract_type={}'.format(symbol_pair, period,contract_type) self.strategy.writeCtaLog('{}开始下载:{} {}数据.URL:{}'.format(datetime.now(), symbol, period, url)) bars = [] content = None try: content = self.session.get(url).content.decode('gbk') bars = execjs.eval(content) except Exception as ex: self.strategy.writeCtaError('exception in get:{},{},{}'.format(url,str(ex), traceback.format_exc())) return False, ret_bars for i, bar in enumerate(bars): if len(bar) < 5: self.strategy.writeCtaError('error when import bar:{}'.format(bar)) return False add_bar = CtaBarData() try: add_bar.vtSymbol = symbol add_bar.symbol = symbol add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000) add_bar.date = add_bar.datetime.strftime('%Y-%m-%d') add_bar.time = add_bar.datetime.strftime('%H:%M:%S') add_bar.tradingDay = add_bar.date add_bar.open = float(bar[1]) add_bar.high = float(bar[2]) add_bar.low = float(bar[3]) add_bar.close = float(bar[4]) add_bar.volume = float(bar[6]) # 这里:5 是交易量,6是交易量转化BTC或LTC数量 except Exception as ex: self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc())) return False, ret_bars if start_dt is not None and bar.datetime < start_dt: continue ret_bars.append(add_bar) if callback is not None: callback(add_bar, bar_is_completed, bar_freq) return True, ret_bars
def getDayBars(self, symbol, callback,start_dt=None): """ 从sina加载最新的Day数据 :param symbol: (全路径得合约名称,先使用ctaTemplate.getFullSymbol() :param callback: 回调函数 :param start_dt: 开始时间,缺省为None :return: 成功/失败 """ sinaBars = [] try: url = u'http://stock.finance.sina.com.cn/futures/api/json.php/InnerFuturesService.getInnerFuturesDailyKLine?symbol={0}'.format(symbol) self.strategy.writeCtaLog(u'从sina下载{0}的日K数据 {1}'.format(symbol, url)) responses = execjs.eval(self.session.get(url).content.decode('gbk')) dayVolume = 0 for item in responses: bar = CtaBarData() bar.vtSymbol = symbol bar.symbol = symbol # bar的close time bar.datetime = datetime.strptime(item['date'], '%Y-%m-%d') if start_dt is not None: if bar.datetime < start_dt: continue bar.date = bar.datetime.strftime('%Y%m%d') bar.tradingDay = bar.date # todo: 需要修改,晚上21点后,修改为next workingday bar.time = bar.datetime.strftime('%H:%M:00') bar.open = float(item['open']) bar.high = float(item['high']) bar.low = float(item['low']) bar.close = float(item['close']) bar.volume = int(item['volume']) bar.dayVolume = bar.volume sinaBars.append(bar) if len(sinaBars)>0: self.strategy.writeCtaLog(u'从sina读取了{0}条日线K数据'.format(len(sinaBars))) # 把sina的bar灌入回调函数 for bar in sinaBars: callback(bar) # 处理完毕,清空 sinaBars = [] return True else: self.strategy.writeCtaLog(u'从sina读取日线K数据失败') return False except Exception as e: self.strategy.writeCtaLog(u'加载Sina历史日线数据失败:'+str(e)) return False
def get_bars(self, symbol, period, callback, bar_is_completed=False,bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] if symbol not in symbol_list: self.strategy.writeCtaError(u'{} {}不在下载清单中'.format(datetime.now(), symbol)) return False, ret_bars if period not in PERIOD_MAPPING: self.strategy.writeCtaError(u'{} 周期{}不在下载清单中'.format(datetime.now(), period)) return False, ret_bars self.strategy.writeCtaLog('{}开始下载:{} {}数据'.format(datetime.now(), symbol, period)) fcoin_period = PERIOD_MAPPING.get(period) fcoin_symbol = symbol.lower().replace('_' , '') try: status,response = self.fcoin.get_candle(resolution=fcoin_period,symbol=fcoin_symbol,limit=200) if not status: return False, ret_bars bars = response.get('data') bars.reverse() #{'seq': 7636160700002, 'low': 6747.96, 'open': 6752.49, 'high': 6753.37, 'base_vol': 356.859787045, # 'quote_vol': 2409051.1568048545, 'id': 1529566740, 'close': 6750.72, 'count': 3673} for i, bar in enumerate(bars): add_bar = CtaBarData() try: add_bar.vtSymbol = symbol add_bar.symbol = symbol add_bar.datetime = datetime.fromtimestamp(bar.get('id',datetime.now()) ) add_bar.date = add_bar.datetime.strftime('%Y-%m-%d') add_bar.time = add_bar.datetime.strftime('%H:%M:%S') add_bar.tradingDay = add_bar.date add_bar.open = float(bar.get('open',0.0)) add_bar.high = float(bar.get('high',0.0)) add_bar.low = float(bar.get('low',0.0)) add_bar.close = float(bar.get('close',0.0)) add_bar.volume = float(bar.get('base_vol',0.0)) except Exception as ex: self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(bar, str(ex), traceback.format_exc())) return False, ret_bars if start_dt is not None and bar.datetime < start_dt: continue ret_bars.append(add_bar) if callback is not None: callback(add_bar, bar_is_completed, bar_freq) return True, ret_bars except Exception as ex: self.strategy.writeCtaError('exception in get bar:{},{}'.format(str(ex), traceback.format_exc())) return False
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:股票 000001.XG period: 周期: 1min,5min,15min,30min,1hour,1day, """ if self.api is None: self.connect() def __select_market_code(code): code = str(code) if code[0] in ['5', '6', '9'] or code[:3] in [ "009", "126", "110", "201", "202", "203", "204" ]: return 1 return 0 # 新版一劳永逸偷懒写法zzz if '.' in symbol: tdx_code, market_str = symbol.split('.') market_code = 1 if market_str.upper() == 'XSHG' else 0 self.symbol_exchange_dict.update({tdx_code: symbol}) # tdx合约与vn交易所的字典 self.symbol_market_dict.update({tdx_code: market_code}) # tdx合约与tdx市场的字典 else: market_code = __select_market_code(symbol) tdx_code = symbol self.symbol_exchange_dict.update({symbol: symbol}) # tdx合约与vn交易所的字典 self.symbol_market_dict.update({symbol: market_code}) # tdx合约与tdx市场的字典 # https://github.com/rainx/pytdx/issues/33 # 0 - 深圳, 1 - 上海 ret_bars = [] if period not in PERIOD_MAPPING.keys(): self.strategy.writeCtaError(u'{} 周期{}不在下载清单中: {}'.format( datetime.now(), period, list(PERIOD_MAPPING.keys()))) # print(u'{} 周期{}不在下载清单中: {}'.format(datetime.now(), period, list(PERIOD_MAPPING.keys()))) return False, ret_bars if self.api is None: return False, ret_bars tdx_period = PERIOD_MAPPING.get(period) if start_dt is None: self.strategy.writeCtaLog(u'没有设置开始时间,缺省为10天前') qry_start_date = datetime.now() - timedelta(days=10) start_dt = qry_start_date else: qry_start_date = start_dt end_date = datetime.now() if qry_start_date > end_date: qry_start_date = end_date self.strategy.writeCtaLog('{}开始下载tdx股票:{} {}数据, {} to {}.'.format( datetime.now(), tdx_code, tdx_period, qry_start_date, end_date)) try: _start_date = end_date _bars = [] _pos = 0 while _start_date > qry_start_date: _res = self.api.get_security_bars( category=PERIOD_MAPPING[period], market=market_code, code=tdx_code, start=_pos, count=QSIZE) if _res is not None: _bars = _res + _bars _pos += QSIZE if _res is not None and len(_res) > 0: _start_date = _res[0]['datetime'] _start_date = datetime.strptime(_start_date, '%Y-%m-%d %H:%M') self.strategy.writeCtaLog( u'分段取数据开始:{}'.format(_start_date)) else: break if len(_bars) == 0: self.strategy.writeCtaError( '{} Handling {}, len1={}..., continue'.format( str(datetime.now()), tdx_code, len(_bars))) return False, ret_bars current_datetime = datetime.now() data = self.api.to_df(_bars) data = data.assign(datetime=pd.to_datetime(data['datetime'])) data = data.assign(ticker=symbol) data['symbol'] = symbol data = data.drop( ['year', 'month', 'day', 'hour', 'minute', 'price', 'ticker'], errors='ignore', axis=1) data = data.rename(index=str, columns={ 'amount': 'volume', }) if len(data) == 0: print('{} Handling {}, len2={}..., continue'.format( str(datetime.now()), tdx_code, len(data))) return False, ret_bars # 通达信是以bar的结束时间标记的,vnpy是以bar开始时间标记的,所以要扣减bar本身的分钟数 data['datetime'] = data['datetime'].apply(lambda x: x - timedelta( minutes=NUM_MINUTE_MAPPING.get(period, 1))) data['trading_date'] = data['datetime'].apply( lambda x: (x.strftime('%Y-%m-%d'))) data['date'] = data['datetime'].apply(lambda x: (x.strftime('%Y-%m-%d'))) data['time'] = data['datetime'].apply(lambda x: (x.strftime('%H:%M:%S'))) data = data.set_index('datetime', drop=False) for index, row in data.iterrows(): add_bar = CtaBarData() try: add_bar.vtSymbol = symbol add_bar.symbol = symbol add_bar.datetime = index add_bar.date = row['date'] add_bar.time = row['time'] add_bar.tradingDay = row['trading_date'] add_bar.open = float(row['open']) add_bar.high = float(row['high']) add_bar.low = float(row['low']) add_bar.close = float(row['close']) add_bar.volume = float(row['volume']) except Exception as ex: self.strategy.writeCtaError( 'error when convert bar:{},ex:{},t:{}'.format( row, str(ex), traceback.format_exc())) # print('error when convert bar:{},ex:{},t:{}'.format(row, str(ex), traceback.format_exc())) return False if start_dt is not None and index < start_dt: continue ret_bars.append(add_bar) if callback is not None: freq = bar_freq bar_is_completed = True if period != '1min' and index == data['datetime'][-1]: # 最后一个bar,可能是不完整的,强制修改 # - 5min修改后freq基本正确 # - 1day在VNPY合成时不关心已经收到多少Bar, 所以影响也不大 # - 但其它分钟周期因为不好精确到每个品种, 修改后的freq可能有错 if index > current_datetime: bar_is_completed = False # 根据秒数算的话,要+1,例如13:31,freq=31,第31根bar freq = NUM_MINUTE_MAPPING[period] - int( (index - current_datetime).total_seconds() / 60) callback(add_bar, bar_is_completed, freq) return True, ret_bars except Exception as ex: self.strategy.writeCtaError('exception in get:{},{},{}'.format( tdx_code, str(ex), traceback.format_exc())) # print('exception in get:{},{},{}'.format(tdx_symbol,str(ex), traceback.format_exc())) self.strategy.writeCtaLog(u'重置连接') TdxData.api = None self.connect() return False, ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] if symbol not in SYMBOL_LIST: self.writeError(u'{} 合约{}不在下载清单中'.format(datetime.now(), symbol)) return False, ret_bars if period not in PERIOD_MAPPING: self.writeError(u'{} 周期{}不在下载清单中'.format(datetime.now(), period)) return False, ret_bars if self.client is None: return False, ret_bars binance_symbol = symbol.upper().replace('_', '') # 转换一下 binance_symbol = symbolFromOtherExchangesToBinance(binance_symbol) binance_period = PERIOD_MAPPING.get(period) self.writeLog('{}开始下载binance:{} {}数据.'.format(datetime.now(), binance_symbol, binance_period)) bars = [] try: bars = self.client.get_klines(symbol=binance_symbol, interval=binance_period) bar_len = len(bars) for i, bar in enumerate(bars): add_bar = CtaBarData() try: add_bar.vtSymbol = symbol add_bar.symbol = symbol add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000) utcdatetime = datetime.utcfromtimestamp(bar[0] / 1e3) add_bar.date = add_bar.datetime.strftime('%Y-%m-%d') add_bar.time = add_bar.datetime.strftime('%H:%M:%S') # 币安的交易日,是按照utc来计算的 add_bar.tradingDay = utcdatetime.strftime('%Y-%m-%d') add_bar.open = float(bar[1]) add_bar.high = float(bar[2]) add_bar.low = float(bar[3]) add_bar.close = float(bar[4]) add_bar.volume = float(bar[5]) ret_bars.append(add_bar) except Exception as ex: self.writeError( 'error when convert bar:{},ex:{},t:{}'.format( bar, str(ex), traceback.format_exc())) return False, ret_bars if start_dt is not None and bar.datetime < start_dt: continue if callback is not None: # 最后一个bar,可能是不完整的,强制修改 if i == bar_len - 1 and bar_is_completed: # 根据秒数算的话,要+1,例如13:31,freq=31,第31根bar freq = int((datetime.now() - add_bar.datetime).total_seconds() / 60) + 1 callback(add_bar, False, freq) else: callback(add_bar, bar_is_completed, bar_freq) return True, ret_bars except Exception as ex: self.writeError('exception in get:{},{},{}'.format( binance_symbol, str(ex), traceback.format_exc())) return False, ret_bars
def getMinBars(self, symbol, callback): bars = [] try: path = 'hisminbar' params = {'id': symbol, 'num': 172, 'period': '15m'} data = self.getData(path, params) if not data: return False barList = [] l = data.split(';') for barStr in l: # 过滤某些空数据 if ',' not in barStr: continue barData = barStr.split(',') d = { 'symbol': barData[0], # 'date': barData[1], # trading day 'tradingDay': barData[1], 'minute': barData[2], 'time': barData[2], 'open': float(barData[3]), 'high': float(barData[4]), 'low': float(barData[5]), 'close': float(barData[6]), 'volume': int(barData[7]), 'openInterest': int(float(barData[8])), 'date': barData[9] # natural day } barList.append(d) barList.reverse() for item in barList: bar = CtaBarData() bar.vtSymbol = symbol bar.symbol = symbol bar.tradingDay = item['tradingDay'] bar.datetime = datetime.strptime( u'{0}-{1}-{2} {3}:{4}:00'.format(item['date'][0:4], item['date'][4:6], item['date'][6:8], item['minute'][0:2], item['minute'][2:4]), '%Y-%m-%d %H:%M:00') bar.time = item['time'] bar.open = item['open'] bar.high = item['high'] bar.low = item['low'] bar.close = item['close'] bar.volume = item['volume'] bar.date = item['date'] if bar.datetime.hour == 10 and bar.datetime.minute == 30: continue # bar.datetime = bar.datetime - timedelta(seconds=5 * 60) callback(bar, bar_is_completed=False, bar_freq=15) # print(u'{0}, {1}, {2}, {3}, {4}'.format(bar.datetime, bar.open, bar.high, bar.low, bar.close)) # bars.append(bar) # print('*' * 20 + 'bars---' + '*' * 20) # print(bars) # if len(bars) > 0: # for bar in bars: # callback(bar) # bars = [] return True # else: # self.strategy.writeCtaLog(u'从shcifo读取分钟数据失败') # return False except Exception as e: self.strategy.writeCtaLog(u'加载shcifo历史分钟数据失败:' + str(e)) return False
def getMinBars(self, symbol, minute, callback,start_dt=None): """ 从sina加载最新的M5,M15,M30,M60数据 :param symbol: (全路径得合约名称,先使用ctaTemplate.getFullSymbol() :param minute: 5,15,30,60 :param callback: 回调函数 :param start_dt: 开始时间,缺省为None :return: 成功/失败 """ if minute not in [5, 15, 30, 60]: return False sinaBars = [] try: url = u'http://stock2.finance.sina.com.cn/futures/api/json.php/InnerFuturesService.getInnerFutures{0}MinKLine?symbol={1}'.format(minute,symbol) self.strategy.writeCtaLog(u'从sina下载{0}的{1}分钟数据 {2}'.format(symbol,minute, url)) responses = execjs.eval(self.session.get(url).content.decode('gbk').split('\n')[-1]) dayVolume = 0 for item in responses: bar = CtaBarData() bar.vtSymbol = symbol bar.symbol = symbol # bar的close time sinaDt = datetime.strptime(item[0], '%Y-%m-%d %H:%M:00') if minute in {5, 15} and sinaDt.hour == 10 and sinaDt.minute == 30: # 这个是sina的bug,它把10:15 ~10:30也包含进来了 continue if minute == 60 and sinaDt.hour in {11,23,1,2} and sinaDt.minute == 30: bar.datetime = sinaDt - timedelta(seconds=(minute /2)* 60) else: bar.datetime = sinaDt - timedelta(seconds=minute * 60) if start_dt is not None: if bar.datetime < start_dt: continue bar.date = bar.datetime.strftime('%Y%m%d') bar.tradingDay = bar.date # todo: 需要修改,晚上21点后,修改为next workingday bar.time = bar.datetime.strftime('%H:%M:00') bar.open = float(item[1]) bar.high = float(item[2]) bar.low = float(item[3]) bar.close = float(item[4]) bar.volume = int(item[5]) # 计算dayvolume if not sinaBars: dayVolume = bar.volume else: if sinaBars[-1].datetime.hour == 14 and bar.datetime.hour !=14: dayVolume = bar.volume else: dayVolume += bar.volume bar.dayVolume = dayVolume sinaBars.append(bar) if len(sinaBars)>0: self.strategy.writeCtaLog(u'从sina读取了{0}条{1}分钟数据'.format(len(sinaBars),minute)) # 把sina的bar灌入回调函数 for bar in sinaBars: callback(bar) # 处理完毕,清空 sinaBars = [] return True else: self.strategy.writeCtaLog(u'从sina读取{0}分钟数据失败'.format(minute)) return False except Exception as e: self.strategy.writeCtaLog(u'加载Sina历史分钟数据失败:'+str(e)) return False
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] symbol = symbol.upper() if symbol not in symbol_list: msg = u'{} {}不在下载清单中'.format(datetime.now(), symbol) if self.strategy: self.strategy.writeCtaError(msg) else: print(msg) return False, ret_bars if period not in period_list: self.writeError(u'{}不在下载时间周期范围:{} 内'.format(period, period_list)) return False, ret_bars candleCount = 200 url = u'https://www.bitmex.com/api/v1/trade/bucketed?binSize={}&partial=false&symbol={}&count={}&reverse=true'.format( period, symbol, candleCount) self.writeLog('{}开始下载:{} {}数据.URL:{}'.format(datetime.now(), symbol, period, url)) content = None try: content = self.session.get(url).content.decode('gbk') except Exception as ex: self.writeError('exception in get:{},{},{}'.format( url, str(ex), traceback.format_exc())) return False, ret_bars bars = execjs.eval(content) if not isinstance(bars, list): self.writeError('返回数据不是list:{}'.format(content)) return False, ret_bars # example bar: # {"timestamp":"2018-08-09T02:37:00.000Z","symbol":"XBTUSD","open":6313.5,"high":6313.5,"low":6313,"close":6313.5, # "trades":63,"volume":243870,"vwap":6313.5299,"lastSize":1000,"turnover":3862773286,"homeNotional":38.62773286, # "foreignNotional":243870} for i, bar in enumerate(bars): add_bar = CtaBarData() try: add_bar.vtSymbol = bar['symbol'] add_bar.symbol = bar['symbol'] raw_datetime = bar['timestamp'] raw_datetime = raw_datetime.split('T') add_bar.date = raw_datetime[0] add_bar.time = raw_datetime[1][:-5] date = add_bar.date.split('-') time = add_bar.time.split(':') add_bar.datetime = datetime(int(date[0]), int(date[1]), int(date[2]), int(time[0]), int(time[1]), int(time[2])) add_bar.tradingDay = add_bar.date add_bar.open = bar['open'] add_bar.high = bar['high'] add_bar.low = bar['low'] add_bar.close = bar['close'] add_bar.volume = bar['volume'] except Exception as ex: self.strategy.writeCtaError( 'error when convert bar:{},ex:{},t:{}'.format( bar, str(ex), traceback.format_exc())) return False, ret_bars if start_dt is not None and bar.datetime < start_dt: continue if callback is not None: callback(add_bar, bar_is_completed, bar_freq) ret_bars.append(add_bar) return True, ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] if period not in period_list: self.strategy.writeCtaError('{} not in {}'.format( period, period_list)) return False, ret_bars period = period.replace('min', 'm').replace('hour', 'h').replace('day', 'd') count = 750 if start_dt is None: if period == '1d': start_dt = datetime.utcnow() - timedelta(days=count - 1) elif period == '1h': print(u'utc now():{}'.format(datetime.utcnow())) start_dt = datetime.utcnow() - timedelta(hours=count - 1) print(u'start_dt():{}'.format(start_dt)) elif period == '5m': start_dt = datetime.utcnow() - timedelta(minutes=(count - 1) * 5) else: start_dt = datetime.utcnow() - timedelta(minutes=count - 1) path = '/trade/bucketed' url = u'https://www.bitmex.com/api/v1' + path params = { 'binSize': period, 'partial': True, 'symbol': symbol, 'startTime': start_dt.strftime('%Y-%m-%d %H:%M'), 'count': count } params = OrderedDict(sorted(params.items())) expires = int(time() + 5) rq = requests.Request(url=url) p = rq.prepare() header = { 'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json', 'api-expires': str(expires), 'api-key': self.apiKey, 'api-signature': self.generateSignature('GET', path, expires, params, body=p.body) } print(header) session = requests.Session() resp = session.request('GET', url, headers=header, params=params, data=None) self.writeLog('{}开始下载:{} {}数据.开始时间:{} URL:{}'.format( datetime.now(), symbol, period, start_dt, url)) bars = None code = resp.status_code bars = resp.json() if code != 200: self.strategy.writeCtaError('get_bars {},{}'.format(code, bars)) return False, ret_bars bar_len = len(bars) self.writeLog(u'共下载:{}条'.format(bar_len)) for i, bar in enumerate(bars): add_bar = CtaBarData() try: add_bar.vtSymbol = symbol add_bar.symbol = symbol utc_str_time = bar.get('timestamp').replace('T', ' ').replace( 'Z', '') utc_datetime = datetime.strptime(utc_str_time, '%Y-%m-%d %H:%M:%S.%f') add_bar.datetime = utc_datetime + timedelta(hours=8) # 时区得调整 +8 if period == '1h': add_bar.datetime = add_bar.datetime - timedelta(hours=1) elif period == '5m': add_bar.datetime = add_bar.datetime - timedelta(minutes=5) elif period == '1m': add_bar.datetime = add_bar.datetime - timedelta(minutes=1) # bitmex是以bar得结束时间,作为bar得datetime,vnpy是以bar得开始时间作为datetime add_bar.date = add_bar.datetime.strftime('%Y-%m-%d') add_bar.time = add_bar.datetime.strftime('%H:%M:%S') add_bar.tradingDay = add_bar.date add_bar.open = float(bar.get('open', 0.0)) add_bar.high = float(bar.get('high', 0.0)) add_bar.low = float(bar.get('low', 0.0)) add_bar.close = float(bar.get('close', 0.0)) add_bar.volume = float(bar.get('volume', 0.0)) ret_bars.append(add_bar) except Exception as ex: self.strategy.writeCtaError( 'error when convert bar:{},ex:{},t:{}'.format( bar, str(ex), traceback.format_exc())) return False, ret_bars if start_dt is not None and add_bar.datetime < start_dt: continue if callback is not None: # 最后一个bar,可能是不完整的,强制修改 if i == bar_len - 1 and bar_is_completed: # 根据秒数算的话,要+1,例如13:31,freq=31,第31根bar freq = int( (datetime.now() - add_bar.datetime).total_seconds() / 60) + 1 callback(add_bar, False, freq) else: callback(add_bar, bar_is_completed, bar_freq) return True, ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] tdx_symbol = symbol.upper().replace('_' , '') tdx_symbol = tdx_symbol.replace('99' , 'L9') if tdx_symbol not in self.symbol_exchange_dict.keys(): self.strategy.writeCtaError(u'{} 合约{}/{}不在下载清单中: {}'.format(datetime.now(), symbol, tdx_symbol, self.symbol_exchange_dict.keys())) return False,ret_bars if period not in PERIOD_MAPPING.keys(): self.strategy.writeCtaError(u'{} 周期{}不在下载清单中: {}'.format(datetime.now(), period, list(PERIOD_MAPPING.keys()))) return False,ret_bars if self.api is None: return False,ret_bars tdx_period = PERIOD_MAPPING.get(period) if start_dt is None: self.strategy.writeCtaLog(u'没有设置开始时间,缺省为10天前') qry_start_date = datetime.now() - timedelta(days=10) else: qry_start_date = start_dt end_date = datetime.combine(datetime.now() + timedelta(days=1),time(ALL_MARKET_END_HOUR, 0)) if qry_start_date > end_date: qry_start_date = end_date self.strategy.writeCtaLog('{}开始下载tdx:{} {}数据, {} to {}.'.format(datetime.now(), tdx_symbol, period, qry_start_date, end_date)) # print('{}开始下载tdx:{} {}数据, {} to {}.'.format(datetime.now(), tdx_symbol, tdx_period, last_date, end_date)) try: _start_date = end_date _bars = [] _pos = 0 while _start_date > qry_start_date: _res = self.api.get_instrument_bars( PERIOD_MAPPING[period], self.symbol_market_dict.get(tdx_symbol,0), tdx_symbol, _pos, QSIZE) if _res is not None: _bars = _res + _bars _pos += QSIZE if _res is not None and len(_res) > 0: _start_date = _res[0]['datetime'] _start_date = datetime.strptime(_start_date, '%Y-%m-%d %H:%M') self.strategy.writeCtaLog(u'分段取数据开始:{}'.format(_start_date)) else: break if len(_bars) == 0: self.strategy.writeCtaError('{} Handling {}, len1={}..., continue'.format( str(datetime.now()), tdx_symbol, len(_bars))) return False, ret_bars current_datetime = datetime.now() data = self.api.to_df(_bars) data = data.assign(datetime=pd.to_datetime(data['datetime'])) data = data.assign(ticker=symbol) data['instrument_id'] = data['ticker'] # if future['market'] == 28 or future['market'] == 47: # # 大写字母: 郑州商品 or 中金所期货 # data['instrument_id'] = data['ticker'] # else: # data['instrument_id'] = data['ticker'].apply(lambda x: x.lower()) data['symbol'] = symbol data = data.drop( ['year', 'month', 'day', 'hour', 'minute', 'price', 'amount', 'ticker'], errors='ignore', axis=1) data = data.rename( index=str, columns={ 'position': 'open_interest', 'trade': 'volume', }) if len(data) == 0: print('{} Handling {}, len2={}..., continue'.format( str(datetime.now()), tdx_symbol, len(data))) return False, ret_bars data['total_turnover'] = data['volume'] data["limit_down"] = 0 data["limit_up"] = 999999 data['trading_date'] = data['datetime'] data['trading_date'] = data['trading_date'].apply(lambda x: (x.strftime('%Y-%m-%d'))) monday_ts = data['datetime'].dt.weekday == 0 # 星期一 night_ts1 = data['datetime'].dt.hour > ALL_MARKET_END_HOUR night_ts2 = data['datetime'].dt.hour < ALL_MARKET_BEGIN_HOUR data.loc[night_ts1, 'datetime'] -= timedelta(days=1) # 所有日期的夜盘(21:00~24:00), 减一天 monday_ts1 = monday_ts & night_ts1 # 星期一的夜盘(21:00~24:00), 再减两天 data.loc[monday_ts1, 'datetime'] -= timedelta(days=2) monday_ts2 = monday_ts & night_ts2 # 星期一的夜盘(00:00~04:00), 再减两天 data.loc[monday_ts2, 'datetime'] -= timedelta(days=2) # data['datetime'] -= timedelta(minutes=1) # 直接给Strategy使用, RiceQuant格式, 不需要减1分钟 data['dt_datetime'] = data['datetime'] data['date'] = data['datetime'].apply(lambda x: (x.strftime('%Y-%m-%d'))) data['time'] = data['datetime'].apply(lambda x: (x.strftime('%H:%M:%S'))) data['datetime'] = data['datetime'].apply(lambda x: float(x.strftime('%Y%m%d%H%M%S'))) data = data.set_index('dt_datetime', drop=False) # data = data[int(last_date.strftime('%Y%m%d%H%M%S')):int(end_date.strftime('%Y%m%d%H%M%S'))] # data = data[str(last_date):str(end_date)] for index, row in data.iterrows(): add_bar = CtaBarData() try: add_bar.vtSymbol = row['symbol'] add_bar.symbol = row['symbol'] add_bar.datetime = index add_bar.date = row['date'] add_bar.time = row['time'] add_bar.tradingDay = row['trading_date'] add_bar.open = float(row['open']) add_bar.high = float(row['high']) add_bar.low = float(row['low']) add_bar.close = float(row['close']) add_bar.volume = float(row['volume']) add_bar.openInterest = float(row['open_interest']) except Exception as ex: self.strategy.writeCtaError('error when convert bar:{},ex:{},t:{}'.format(row, str(ex), traceback.format_exc())) # print('error when convert bar:{},ex:{},t:{}'.format(row, str(ex), traceback.format_exc())) return False if start_dt is not None and index < start_dt: continue ret_bars.append(add_bar) if callback is not None: freq = bar_freq bar_is_completed = True if period != '1min' and index == data['dt_datetime'][-1]: # 最后一个bar,可能是不完整的,强制修改 # - 5min修改后freq基本正确 # - 1day在VNPY合成时不关心已经收到多少Bar, 所以影响也不大 # - 但其它分钟周期因为不好精确到每个品种, 修改后的freq可能有错 if index > current_datetime: bar_is_completed = False # 根据秒数算的话,要+1,例如13:31,freq=31,第31根bar freq = NUM_MINUTE_MAPPING[period] - int((index - current_datetime).total_seconds() / 60) callback(add_bar, bar_is_completed, freq) return True, ret_bars except Exception as ex: self.strategy.writeCtaError('exception in get:{},{},{}'.format(tdx_symbol,str(ex), traceback.format_exc())) # print('exception in get:{},{},{}'.format(tdx_symbol,str(ex), traceback.format_exc())) self.strategy.writeCtaLog(u'重置连接') TdxFutureData.api = None self.connect() return False,ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] if symbol not in symbol_list: self.strategy.writeCtaError(u'{} {}不在下载清单中'.format( datetime.now(), symbol)) return False, ret_bars group_sec, range_hour = self.get_sec_hours(period) url = u'https://data.gateio.io/api2/1/candlestick2/{}?group_sec={}&range_hour={}'.format( symbol, group_sec, range_hour) self.strategy.writeCtaLog('{}开始下载:{} {}数据.URL:{}'.format( datetime.now(), symbol, period, url)) content = None try: content = self.session.get(url).content.decode('gbk') except Exception as ex: self.strategy.writeCtaError('exception in get:{},{},{}'.format( url, str(ex), traceback.format_exc())) return False, ret_bars content = execjs.eval(content) if not isinstance(content, dict): self.strategy.writeCtaError( u'content is not dict format:{}'.format(content)) return False, ret_bars bars = content.get('data', []) for i, bar in enumerate(bars): if len(bar) < 6: self.strategy.writeCtaError( 'error when import bar:{}'.format(bar)) return False, ret_bars add_bar = CtaBarData() try: add_bar.vtSymbol = symbol add_bar.symbol = symbol add_bar.datetime = datetime.fromtimestamp(int(bar[0]) / 1000) add_bar.date = add_bar.datetime.strftime('%Y-%m-%d') add_bar.time = add_bar.datetime.strftime('%H:%M:%S') add_bar.tradingDay = add_bar.date add_bar.volume = float(bar[1]) add_bar.close = float(bar[2]) add_bar.high = float(bar[3]) add_bar.low = float(bar[4]) add_bar.open = float(bar[5]) except Exception as ex: self.strategy.writeCtaError( 'error when convert bar:{},ex:{},t:{}'.format( bar, str(ex), traceback.format_exc())) return False, ret_bars if start_dt is not None and bar.datetime < start_dt: continue ret_bars.append(add_bar) if callback is not None: callback(add_bar, bar_is_completed, bar_freq) return True, ret_bars
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] if symbol not in SYMBOL_LIST: self.strategy.writeCtaError(u'{} 合约{}不在下载清单中'.format( datetime.now(), symbol)) return False, ret_bars if period not in PERIOD_MAPPING: self.strategy.writeCtaError(u'{} 周期{}不在下载清单中'.format( datetime.now(), period)) return False, ret_bars if self.client is None: return False, ret_bars binance_symbol = symbol.upper().replace('_', '') binance_period = PERIOD_MAPPING.get(period) self.strategy.writeCtaLog('{}开始下载binance:{} {}数据.'.format( datetime.now(), binance_symbol, binance_period)) try: bars = self.client.get_klines(symbol=binance_symbol, interval=binance_period) for i, bar in enumerate(bars): add_bar = CtaBarData() try: add_bar.vtSymbol = symbol add_bar.symbol = symbol add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000) add_bar.date = add_bar.datetime.strftime('%Y-%m-%d') add_bar.time = add_bar.datetime.strftime('%H:%M:%S') add_bar.tradingDay = add_bar.date add_bar.open = float(bar[1]) add_bar.high = float(bar[2]) add_bar.low = float(bar[3]) add_bar.close = float(bar[4]) add_bar.volume = float(bar[5]) except Exception as ex: self.strategy.writeCtaError( 'error when convert bar:{},ex:{},t:{}'.format( bar, str(ex), traceback.format_exc())) return False if start_dt is not None and bar.datetime < start_dt: continue ret_bars.append(add_bar) if callback is not None: callback(add_bar, bar_is_completed, bar_freq) return True, ret_bars except Exception as ex: self.strategy.writeCtaError('exception in get:{},{},{}'.format( binance_symbol, str(ex), traceback.format_exc())) return False, ret_bars
def getMinBars2(self, symbol, minute, callback): """# 从sina加载最新的M5,M15,M30,M60数据(针对中金所)""" if minute not in {5, 15, 30, 60}: return False sinaBars = [] try: timestamp = (datetime.utcnow() - datetime(1970, 1, 1)).total_seconds() url = u'http://stock2.finance.sina.com.cn/futures/api/jsonp.php/var%20_{1}_{0}_{2}=/InnerFuturesNewService.getFewMinLine?symbol={1}&type={0}'.format( minute, symbol, timestamp) #url = u'http://stock2.finance.sina.com.cn/futures/api/json.php/InnerFuturesService.getInnerFutures{0}MinKLine?symbol={1}'.format(minute,symbol) self.strategy.writeCtaLog(u'从sina下载{0}的{1}分钟数据 {2}'.format( symbol, minute, url)) response_data = self.session.get(url).content response_data = response_data.decode('gbk').split('=')[-1] response_data = response_data.replace('(', '') response_data = response_data.replace(');', '') responses = execjs.eval(response_data) dayVolume = 0 for item in responses: bar = CtaBarData() bar.vtSymbol = symbol bar.symbol = symbol # bar的close time sinaDt = datetime.strptime(item['d'], '%Y-%m-%d %H:%M:00') if minute in {5, 15 } and sinaDt.hour == 10 and sinaDt.minute == 30: # 这个是sina的bug,它把10:15 ~10:30也包含进来了 continue if minute == 60 and sinaDt.hour in {11, 23, 1, 2 } and sinaDt.minute == 30: bar.datetime = sinaDt - timedelta(seconds=(minute / 2) * 60) else: bar.datetime = sinaDt - timedelta(seconds=minute * 60) bar.date = bar.datetime.strftime('%Y%m%d') bar.tradingDay = bar.date # todo: 需要修改,晚上21点后,修改为next workingday bar.time = bar.datetime.strftime('%H:%M:00') bar.open = float(item['o']) bar.high = float(item['h']) bar.low = float(item['l']) bar.close = float(item['c']) bar.volume = int(item['v']) # 计算dayvolume if not sinaBars: dayVolume = bar.volume else: if sinaBars[ -1].datetime.hour == 14 and bar.datetime.hour != 14: dayVolume = bar.volume else: dayVolume += bar.volume bar.dayVolume = dayVolume sinaBars.append(bar) if len(sinaBars) > 0: self.strategy.writeCtaLog(u'从sina读取了{0}条{1}分钟数据'.format( len(sinaBars), minute)) # 把sina的bar灌入回调函数 for bar in sinaBars: callback(bar) # 处理完毕,清空 sinaBars = [] return True else: self.strategy.writeCtaLog(u'从sina读取{0}分钟数据失败'.format(minute)) return False except Exception as e: self.strategy.writeCtaLog(u'加载Sina历史分钟数据失败:' + str(e)) return False
def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] symbol_pair = systemSymbolToVnSymbol(symbol) if symbol_pair not in symbol_list: msg = u'{} {}不在下载清单中'.format(datetime.now(), symbol_pair) if self.strategy: self.strategy.writeCtaError(msg) else: print(msg) return False, ret_bars symbol = symbol_pair.replace('_', '') if period == '1hour': period = '60min' if period not in period_list: self.writeError(u'{}不在下载时间周期范围:{} 内'.format(period, period_list)) return False, ret_bars url = u'https://api.huobipro.com/market/history/kline?symbol={}&period={}& size=2000&AccessKeyId=fff-xxx-ssss-kkk'.format( symbol, period) self.writeLog('{}开始下载:{} {}数据.URL:{}'.format(datetime.now(), symbol_pair, period, url)) content = None try: content = self.session.get(url).content.decode('gbk') except Exception as ex: self.writeError('exception in get:{},{},{}'.format( url, str(ex), traceback.format_exc())) return False, ret_bars ret_dict = execjs.eval(content) bars = ret_dict.get('data', None) if not isinstance(bars, list): self.writeError('返回数据不是list:{}'.format(content)) return False, ret_bars bars.reverse() for i, bar in enumerate(bars): add_bar = CtaBarData() try: add_bar.vtSymbol = symbol add_bar.symbol = symbol add_bar.datetime = datetime.fromtimestamp(bar[0] / 1000) add_bar.date = add_bar.datetime.strftime('%Y-%m-%d') add_bar.time = add_bar.datetime.strftime('%H:%M:%S') add_bar.tradingDay = add_bar.date add_bar.open = float(bar[1]) add_bar.high = float(bar[2]) add_bar.low = float(bar[3]) add_bar.close = float(bar[4]) add_bar.volume = float(bar[5]) except Exception as ex: self.strategy.writeCtaError( 'error when convert bar:{},ex:{},t:{}'.format( bar, str(ex), traceback.format_exc())) return False, ret_bars if start_dt is not None and bar.datetime < start_dt: continue if callback is not None: callback(add_bar, bar_is_completed, bar_freq) return True, ret_bars