def __init__(self, path): if not os.path.exists(path): raise RuntimeError('bundle path {} not exist'.format(os.path.abspath(path))) def _p(name): return os.path.join(path, name) self._day_bars = [ DayBarStore(_p('stocks.bcolz'), StockBarConverter), DayBarStore(_p('indexes.bcolz'), IndexBarConverter), DayBarStore(_p('futures.bcolz'), FutureDayBarConverter), DayBarStore(_p('funds.bcolz'), FundDayBarConverter), ] self._instruments = InstrumentStore(_p('instruments.pk')) self._dividends = DividendStore(_p('original_dividends.bcolz')) self._trading_dates = TradingDatesStore(_p('trading_dates.bcolz')) self._yield_curve = YieldCurveStore(_p('yield_curve.bcolz')) self._split_factor = SimpleFactorStore(_p('split_factor.bcolz')) self._ex_cum_factor = SimpleFactorStore(_p('ex_cum_factor.bcolz')) self._st_stock_days = DateSet(_p('st_stock_days.bcolz')) self._suspend_days = DateSet(_p('suspended_days.bcolz')) self.get_yield_curve = self._yield_curve.get_yield_curve self.get_risk_free_rate = self._yield_curve.get_risk_free_rate if os.path.exists(_p('public_funds.bcolz')): self._day_bars.append(DayBarStore(_p('public_funds.bcolz'), PublicFundDayBarConverter)) self._public_fund_dividends = DividendStore(_p('public_fund_dividends.bcolz')) self._non_subscribable_days = DateSet(_p('non_subscribable_days.bcolz')) self._non_redeemable_days = DateSet(_p('non_redeemable_days.bcolz'))
def __init__(self, parent=None): self._instruments = { i.order_book_id: i for i in InstrumentStore(_p( "instruments.pk")).get_all_instruments() } self._sym_id_map = { i.symbol: k for k, i in six.iteritems(self._instruments) # 过滤掉 CSI300, SSE50, CSI500, SSE180 if not i.order_book_id.endswith('INDX') } # 股票ST数据 self._st_stock_days = DateSet(_p('st_stock_days.bcolz')) # 股票停牌数据 self._suspended_days = DateSet(_p('suspended_days.bcolz')) try: # FIXME # 沪深300 中证500 固定使用上证的 for o in ['000300.XSHG', '000905.XSHG']: self._sym_id_map[self._instruments[o].symbol] = o # 上证180 及 上证180指数 两个symbol都指向 000010.XSHG self._sym_id_map[ self._instruments['SSE180.INDX'].symbol] = '000010.XSHG' except KeyError: pass
def __init__(self, parent=None): super(TProService, self).__init__(parent) self._api = None # 数据接口 self._dbClient = None # MongoDB接口 self._active = False # 运行状态 self.thread = Thread(target=self.run) def _p(name): return os.path.join(os.path.expanduser("~/.rqalpha"), "bundle", name) # 合约列表 self._instruments = [ x for x in InstrumentStore(_p( 'instruments.pk')).get_all_instruments() if x.type in ('CS', 'INDX') and x.exchange in ('XSHE', 'XSHG') ] self._total_of_instrument = len(self._instruments) # 交易日历 try: self._trading_calendar = pd.read_csv('etc/calendar.csv').trade_date except Exception: self._trading_calendar = None # 股票ST信息 self._st_stock_days = DateSet(_p('st_stock_days.bcolz')) # 股票停牌信息 self._suspend_days = DateSet(_p('suspended_days.bcolz'))
def __init__(self, path): if not os.path.exists(path): raise RuntimeError('bundle path {} not exist'.format( os.path.abspath(path))) def _p(name): return os.path.join(path, name) self._market = Environment.get_instance().config.base.market if self._market == MARKET.CN: self._day_bars = [ DayBarStore(_p('stocks.bcolz'), StockBarConverter), DayBarStore(_p('indexes.bcolz'), IndexBarConverter), DayBarStore(_p('futures.bcolz'), FutureDayBarConverter), DayBarStore(_p('funds.bcolz'), FundDayBarConverter), ] self._instruments = InstrumentStore(_p('instruments.pk')) self._dividends = DividendStore(_p('original_dividends.bcolz')) self._trading_dates = TradingDatesStore(_p('trading_dates.bcolz')) self._yield_curve = YieldCurveStore(_p('yield_curve.bcolz')) self._split_factor = SimpleFactorStore(_p('split_factor.bcolz')) self._ex_cum_factor = SimpleFactorStore(_p('ex_cum_factor.bcolz')) self._st_stock_days = DateSet(_p('st_stock_days.bcolz')) self._suspend_days = DateSet(_p('suspended_days.bcolz')) self.get_yield_curve = self._yield_curve.get_yield_curve self.get_risk_free_rate = self._yield_curve.get_risk_free_rate if os.path.exists(_p('public_funds.bcolz')): self._day_bars.append( DayBarStore(_p('public_funds.bcolz'), PublicFundDayBarConverter)) self._public_fund_dividends = DividendStore( _p('public_fund_dividends.bcolz')) self._non_subscribable_days = DateSet( _p('non_subscribable_days.bcolz')) self._non_redeemable_days = DateSet( _p('non_redeemable_days.bcolz')) elif self._market == MARKET.HK: self._day_bars = [ HkDayBarStore(_p("hk_stocks.bcolz"), HkStockBarConverter) ] self._instruments = InstrumentStore(_p("hk_instruments.pk")) self._dividends = HkDividendStore(_p('hk_dividend.bcolz')) self._trading_dates = TradingDatesStore( _p("hk_trading_dates.bcolz")) self._yield_curve = HkYieldCurveMocker() self._split_factor = SimpleFactorStore(_p('hk_split_factor.bcolz')) self._ex_cum_factor = SimpleFactorStore( _p("hk_ex_cum_factor.bcolz")) self._suspend_days = DateSet(_p('hk_suspended_days.bcolz')) else: raise NotImplementedError
def __init__(self, path): if not os.path.exists(path): raise RuntimeError('bundle path {} not exist'.format(os.path.abspath(path))) def _p(name): return os.path.join(path, name) self._day_bars = [ DayBarStore(_p('stocks.bcolz'), StockBarConverter), DayBarStore(_p('indexes.bcolz'), IndexBarConverter), DayBarStore(_p('futures.bcolz'), FutureDayBarConverter), DayBarStore(_p('funds.bcolz'), FundDayBarConverter), ] self._instruments = InstrumentStore(_p('instruments.pk')) self._dividends = DividendStore(_p('original_dividends.bcolz')) self._trading_dates = TradingDatesStore(_p('trading_dates.bcolz')) self._yield_curve = YieldCurveStore(_p('yield_curve.bcolz')) self._split_factor = SimpleFactorStore(_p('split_factor.bcolz')) self._ex_cum_factor = SimpleFactorStore(_p('ex_cum_factor.bcolz')) self._share_transformation = ShareTransformationStore(_p('share_transformation.json')) self._st_stock_days = DateSet(_p('st_stock_days.bcolz')) self._suspend_days = DateSet(_p('suspended_days.bcolz')) self.get_yield_curve = self._yield_curve.get_yield_curve self.get_risk_free_rate = self._yield_curve.get_risk_free_rate if os.path.exists(_p('public_funds.bcolz')): self._day_bars.append(DayBarStore(_p('public_funds.bcolz'), PublicFundDayBarConverter)) self._public_fund_dividends = DividendStore(_p('public_fund_dividends.bcolz')) self._non_subscribable_days = DateSet(_p('non_subscribable_days.bcolz')) self._non_redeemable_days = DateSet(_p('non_redeemable_days.bcolz'))
class BaseDataSource(AbstractDataSource): def __init__(self, path): if not os.path.exists(path): raise RuntimeError('bundle path {} not exist'.format( os.path.abspath(path))) def _p(name): return os.path.join(path, name) self._day_bars = [ DayBarStore(_p('stocks.bcolz'), StockBarConverter), DayBarStore(_p('indexes.bcolz'), IndexBarConverter), DayBarStore(_p('futures.bcolz'), FutureDayBarConverter), DayBarStore(_p('funds.bcolz'), FundDayBarConverter), ] self._instruments = InstrumentStore(_p('instruments.pk')) self._dividends = DividendStore(_p('original_dividends.bcolz')) self._trading_dates = TradingDatesStore(_p('trading_dates.bcolz')) self._yield_curve = YieldCurveStore(_p('yield_curve.bcolz')) self._split_factor = SimpleFactorStore(_p('split_factor.bcolz')) self._ex_cum_factor = SimpleFactorStore(_p('ex_cum_factor.bcolz')) self._st_stock_days = DateSet(_p('st_stock_days.bcolz')) self._suspend_days = DateSet(_p('suspended_days.bcolz')) self.get_yield_curve = self._yield_curve.get_yield_curve self.get_risk_free_rate = self._yield_curve.get_risk_free_rate if os.path.exists(_p('public_funds.bcolz')): self._day_bars.append( DayBarStore(_p('public_funds.bcolz'), PublicFundDayBarConverter)) self._public_fund_dividends = DividendStore( _p('public_fund_dividends.bcolz')) self._non_subscribable_days = DateSet( _p('non_subscribable_days.bcolz')) self._non_redeemable_days = DateSet( _p('non_redeemable_days.bcolz')) def get_dividend(self, order_book_id, public_fund=False): if public_fund: return self._public_fund_dividends.get_dividend(order_book_id) return self._dividends.get_dividend(order_book_id) def get_trading_minutes_for(self, order_book_id, trading_dt): raise NotImplementedError def get_trading_calendar(self): return self._trading_dates.get_trading_calendar() def get_all_instruments(self): return self._instruments.get_all_instruments() def is_suspended(self, order_book_id, dates): return self._suspend_days.contains(order_book_id, dates) def is_st_stock(self, order_book_id, dates): return self._st_stock_days.contains(order_book_id, dates) INSTRUMENT_TYPE_MAP = { 'CS': 0, 'INDX': 1, 'Future': 2, 'ETF': 3, 'LOF': 3, 'FenjiA': 3, 'FenjiB': 3, 'FenjiMu': 3, 'PublicFund': 4 } def _index_of(self, instrument): return self.INSTRUMENT_TYPE_MAP[instrument.type] @lru_cache(None) def _all_day_bars_of(self, instrument): i = self._index_of(instrument) return self._day_bars[i].get_bars(instrument.order_book_id, fields=None) @lru_cache(None) def _filtered_day_bars(self, instrument): bars = self._all_day_bars_of(instrument) if bars is None: return None return bars[bars['volume'] > 0] def get_bar(self, instrument, dt, frequency): if frequency != '1d': raise NotImplementedError bars = self._all_day_bars_of(instrument) if bars is None: return dt = np.uint64(convert_date_to_int(dt)) pos = bars['datetime'].searchsorted(dt) if pos >= len(bars) or bars['datetime'][pos] != dt: return None return bars[pos] def get_settle_price(self, instrument, date): bar = self.get_bar(instrument, date, '1d') if bar is None: return np.nan return bar['settlement'] @staticmethod def _are_fields_valid(fields, valid_fields): if fields is None: return True if isinstance(fields, six.string_types): return fields in valid_fields for field in fields: if field not in valid_fields: return False return True def get_ex_cum_factor(self, order_book_id): return self._ex_cum_factor.get_factors(order_book_id) def history_bars(self, instrument, bar_count, frequency, fields, dt, skip_suspended=True, include_now=False, adjust_type='pre', adjust_orig=None): if frequency != '1d': raise NotImplementedError if skip_suspended and instrument.type == 'CS': bars = self._filtered_day_bars(instrument) else: bars = self._all_day_bars_of(instrument) if bars is None or not self._are_fields_valid(fields, bars.dtype.names): return None dt = np.uint64(convert_date_to_int(dt)) i = bars['datetime'].searchsorted(dt, side='right') left = i - bar_count if i >= bar_count else 0 bars = bars[left:i] if adjust_type == 'none' or instrument.type in {'Future', 'INDX'}: # 期货及指数无需复权 return bars if fields is None else bars[fields] if isinstance(fields, str) and fields not in FIELDS_REQUIRE_ADJUSTMENT: return bars if fields is None else bars[fields] return adjust_bars(bars, self.get_ex_cum_factor(instrument.order_book_id), fields, adjust_type, adjust_orig) def get_yield_curve(self, start_date, end_date, tenor=None): return self._yield_curve.get_yield_curve(start_date, end_date, tenor) def get_risk_free_rate(self, start_date, end_date): return self._yield_curve.get_risk_free_rate(start_date, end_date) def current_snapshot(self, instrument, frequency, dt): raise NotImplementedError def get_split(self, order_book_id): return self._split_factor.get_factors(order_book_id) def available_data_range(self, frequency): if frequency in ['tick', '1d']: s, e = self._day_bars[ self.INSTRUMENT_TYPE_MAP['INDX']].get_date_range('000001.XSHG') return convert_int_to_date(s).date(), convert_int_to_date(e).date() raise NotImplementedError def get_margin_info(self, instrument): return { 'margin_type': MARGIN_TYPE.BY_MONEY, 'long_margin_ratio': instrument.margin_rate, 'short_margin_ratio': instrument.margin_rate, } def get_commission_info(self, instrument): return CN_FUTURE_INFO[instrument.underlying_symbol]['speculation'] def get_ticks(self, order_book_id, date): raise NotImplementedError def public_fund_commission(self, instrument, buy): if buy: return PUBLIC_FUND_COMMISSION[instrument.fund_type]['Buy'] else: return PUBLIC_FUND_COMMISSION[instrument.fund_type]['Sell'] def non_subscribable(self, order_book_id, dates): return self._non_subscribable_days.contains(order_book_id, dates) def non_redeemable(self, order_book_id, dates): return self._non_redeemable_days.contains(order_book_id, dates) def get_tick_size(self, instrument): if instrument.type in ['CS', 'INDX']: return 0.01 elif instrument.type in ['ETF', 'LOF', 'FenjiB', 'FenjiA', 'FenjiMu']: return 0.001 elif instrument.type == 'Future': return CN_FUTURE_INFO[ instrument.underlying_symbol]['speculation']['tick_size'] else: # NOTE: you can override get_tick_size in your custom data source raise RuntimeError(_("Unsupported instrument type for tick size"))
class BaseDataSource(AbstractDataSource): def __init__(self, path): if not os.path.exists(path): raise RuntimeError('bundle path {} not exist'.format(os.path.abspath(path))) def _p(name): return os.path.join(path, name) self._day_bars = [ DayBarStore(_p('stocks.bcolz'), StockBarConverter), DayBarStore(_p('indexes.bcolz'), IndexBarConverter), DayBarStore(_p('futures.bcolz'), FutureDayBarConverter), DayBarStore(_p('funds.bcolz'), FundDayBarConverter), ] self._instruments = InstrumentStore(_p('instruments.pk')) self._dividends = DividendStore(_p('original_dividends.bcolz')) self._trading_dates = TradingDatesStore(_p('trading_dates.bcolz')) self._yield_curve = YieldCurveStore(_p('yield_curve.bcolz')) self._split_factor = SimpleFactorStore(_p('split_factor.bcolz')) self._ex_cum_factor = SimpleFactorStore(_p('ex_cum_factor.bcolz')) self._st_stock_days = DateSet(_p('st_stock_days.bcolz')) self._suspend_days = DateSet(_p('suspended_days.bcolz')) self.get_yield_curve = self._yield_curve.get_yield_curve self.get_risk_free_rate = self._yield_curve.get_risk_free_rate if os.path.exists(_p('public_funds.bcolz')): self._day_bars.append(DayBarStore(_p('public_funds.bcolz'), PublicFundDayBarConverter)) self._public_fund_dividends = DividendStore(_p('public_fund_dividends.bcolz')) self._non_subscribable_days = DateSet(_p('non_subscribable_days.bcolz')) self._non_redeemable_days = DateSet(_p('non_redeemable_days.bcolz')) def get_dividend(self, order_book_id, public_fund=False): if public_fund: return self._public_fund_dividends.get_dividend(order_book_id) return self._dividends.get_dividend(order_book_id) def get_trading_minutes_for(self, order_book_id, trading_dt): raise NotImplementedError def get_trading_calendar(self): return self._trading_dates.get_trading_calendar() def get_all_instruments(self): return self._instruments.get_all_instruments() def is_suspended(self, order_book_id, dates): return self._suspend_days.contains(order_book_id, dates) def is_st_stock(self, order_book_id, dates): return self._st_stock_days.contains(order_book_id, dates) INSTRUMENT_TYPE_MAP = { 'CS': 0, 'INDX': 1, 'Future': 2, 'ETF': 3, 'LOF': 3, 'FenjiA': 3, 'FenjiB': 3, 'FenjiMu': 3, 'PublicFund': 4 } def _index_of(self, instrument): return self.INSTRUMENT_TYPE_MAP[instrument.type] @lru_cache(None) def _all_day_bars_of(self, instrument): i = self._index_of(instrument) return self._day_bars[i].get_bars(instrument.order_book_id, fields=None) @lru_cache(None) def _filtered_day_bars(self, instrument): bars = self._all_day_bars_of(instrument) if bars is None: return None return bars[bars['volume'] > 0] def get_bar(self, instrument, dt, frequency): if frequency != '1d': raise NotImplementedError bars = self._all_day_bars_of(instrument) if bars is None: return dt = np.uint64(convert_date_to_int(dt)) pos = bars['datetime'].searchsorted(dt) if pos >= len(bars) or bars['datetime'][pos] != dt: return None return bars[pos] def get_settle_price(self, instrument, date): bar = self.get_bar(instrument, date, '1d') if bar is None: return np.nan return bar['settlement'] @staticmethod def _are_fields_valid(fields, valid_fields): if fields is None: return True if isinstance(fields, six.string_types): return fields in valid_fields for field in fields: if field not in valid_fields: return False return True def get_ex_cum_factor(self, order_book_id): return self._ex_cum_factor.get_factors(order_book_id) def history_bars(self, instrument, bar_count, frequency, fields, dt, skip_suspended=True, include_now=False, adjust_type='pre', adjust_orig=None): if frequency != '1d': raise NotImplementedError if skip_suspended and instrument.type == 'CS': bars = self._filtered_day_bars(instrument) else: bars = self._all_day_bars_of(instrument) if bars is None or not self._are_fields_valid(fields, bars.dtype.names): return None dt = np.uint64(convert_date_to_int(dt)) i = bars['datetime'].searchsorted(dt, side='right') left = i - bar_count if i >= bar_count else 0 bars = bars[left:i] if adjust_type == 'none' or instrument.type in {'Future', 'INDX'}: # 期货及指数无需复权 return bars if fields is None else bars[fields] if isinstance(fields, str) and fields not in FIELDS_REQUIRE_ADJUSTMENT: return bars if fields is None else bars[fields] return adjust_bars(bars, self.get_ex_cum_factor(instrument.order_book_id), fields, adjust_type, adjust_orig) def get_yield_curve(self, start_date, end_date, tenor=None): return self._yield_curve.get_yield_curve(start_date, end_date, tenor) def get_risk_free_rate(self, start_date, end_date): return self._yield_curve.get_risk_free_rate(start_date, end_date) def current_snapshot(self, instrument, frequency, dt): raise NotImplementedError def get_split(self, order_book_id): return self._split_factor.get_factors(order_book_id) def available_data_range(self, frequency): if frequency in ['tick', '1d']: s, e = self._day_bars[self.INSTRUMENT_TYPE_MAP['INDX']].get_date_range('000001.XSHG') return convert_int_to_date(s).date(), convert_int_to_date(e).date() raise NotImplementedError def get_margin_info(self, instrument): return { 'margin_type': MARGIN_TYPE.BY_MONEY, 'long_margin_ratio': instrument.margin_rate, 'short_margin_ratio': instrument.margin_rate, } def get_commission_info(self, instrument): return CN_FUTURE_INFO[instrument.underlying_symbol]['speculation'] def get_ticks(self, order_book_id, date): raise NotImplementedError def public_fund_commission(self, instrument, buy): if buy: return PUBLIC_FUND_COMMISSION[instrument.fund_type]['Buy'] else: return PUBLIC_FUND_COMMISSION[instrument.fund_type]['Sell'] def non_subscribable(self, order_book_id, dates): return self._non_subscribable_days.contains(order_book_id, dates) def non_redeemable(self, order_book_id, dates): return self._non_redeemable_days.contains(order_book_id, dates) def get_tick_size(self, instrument): if instrument.type in ['CS', 'INDX']: return 0.01 elif instrument.type in ['ETF', 'LOF', 'FenjiB', 'FenjiA', 'FenjiMu']: return 0.001 elif instrument.type == 'Future': return CN_FUTURE_INFO[instrument.underlying_symbol]['speculation']['tick_size'] else: # NOTE: you can override get_tick_size in your custom data source raise RuntimeError(_("Unsupported instrument type for tick size"))
class InstrumentsMixin(object): """docstring for InstrumentsMixin""" def __init__(self, parent=None): self._instruments = { i.order_book_id: i for i in InstrumentStore(_p( "instruments.pk")).get_all_instruments() } self._sym_id_map = { i.symbol: k for k, i in six.iteritems(self._instruments) # 过滤掉 CSI300, SSE50, CSI500, SSE180 if not i.order_book_id.endswith('INDX') } # 股票ST数据 self._st_stock_days = DateSet(_p('st_stock_days.bcolz')) # 股票停牌数据 self._suspended_days = DateSet(_p('suspended_days.bcolz')) try: # FIXME # 沪深300 中证500 固定使用上证的 for o in ['000300.XSHG', '000905.XSHG']: self._sym_id_map[self._instruments[o].symbol] = o # 上证180 及 上证180指数 两个symbol都指向 000010.XSHG self._sym_id_map[ self._instruments['SSE180.INDX'].symbol] = '000010.XSHG' except KeyError: pass def industry(self, code): # 按行业获取合约代码 return [ v.order_book_id for v in self._instruments.values() if v.type == 'CS' and v.industry_code == code ] def all_instruments(self, types, dt=None): return [ i for i in self._instruments.values() if ((dt is None or i.listed_date.date() <= dt.date() <= i. de_listed_date.date()) and (types is None or i.type in types)) ] def _instrument(self, sym_or_id): try: return self._instruments[sym_or_id] except KeyError: try: sym_or_id = self._sym_id_map[sym_or_id] return self._instruments[sym_or_id] except KeyError: return None def instruments(self, sym_or_ids): if isinstance(sym_or_ids, six.string_types): return self._instrument(sym_or_ids) return [ i for i in [self._instrument(sid) for sid in sym_or_ids] if i is not None ] def get_future_contracts(self, underlying, date): date = date.replace(hour=0, minute=0, second=0) futures = [ v for o, v in six.iteritems(self._instruments) if v.type == 'Future' and v.underlying_symbol == underlying and not o.endswith('88') and not o.endswith('99') ] if not futures: return [] return sorted(i.order_book_id for i in futures if i.listed_date <= date <= i.de_listed_date) def get_stock_contracts(self): return [ v for o, v in six.iteritems(self._instruments) if v.type in ['CS', 'INDX'] and v.exchange in ['XSHE', 'XSHG'] and re.match(r'^\d', o) and o not in IGNORE_INDEX ] def is_suspended(self, order_book_id, dates): """ 根据 dates 来获取对应合约停牌的日期 :param str order_book_id :param list['int' | 'np.int64' | 'np.uint64' | 'pd.Timestamp'] dates: 日期列表 :return: list[bool] """ if not isinstance(dates, list): dates = [dates] return self._suspended_days.contains(order_book_id, dates) def is_st_stock(self, order_book_id, dates): """ 根据 dates 来获取对应合约被ST处理的日期 :param str symbol: 合约代码(Jaqs格式) :param list['int' | 'np.int64' | 'np.uint64' | 'pd.Timestamp'] dates: 日期列表 :return: list['int'] """ if not isinstance(dates, list): dates = [dates] return self._st_stock_days.contains(order_book_id, dates)