def gen_future_info(d): future_info_file = os.path.join(d, 'future_info.json') if not os.path.exists(future_info_file): init_future_info(d) return with open(future_info_file, 'r') as f: all_futures_info = json.load(f) future_list = [] symbol_list = [] param = [ 'close_commission_ratio', 'close_commission_today_ratio', 'commission_type', 'open_commission_ratio' ] for i in all_futures_info: if i.get('order_book_id'): future_list.append(i.get('order_book_id')) else: symbol_list.append(i.get('underlying_symbol')) futures_order_book_id = rqdatac.all_instruments( type='Future')['order_book_id'].unique() for future in futures_order_book_id: underlying_symbol = re.match(r'^[a-zA-Z]*', future).group() if future in future_list: continue future_dict = {} commission = rqdatac.futures.get_commission_margin(future) if not commission.empty: future_list.append(future) future_dict['order_book_id'] = future commission = commission.iloc[0] for p in param: future_dict[p] = commission[p] future_dict['tick_size'] = rqdatac.instruments(future).tick_size() elif underlying_symbol in symbol_list \ or underlying_symbol in {'S', 'TC', 'ER', 'WS', 'WT', 'RO', 'ME'}: continue else: symbol_list.append(underlying_symbol) future_dict['underlying_symbol'] = underlying_symbol try: dominant = rqdatac.futures.get_dominant( underlying_symbol).iloc[-1] except AttributeError: # FIXME: why get_dominant return None??? continue commission = rqdatac.futures.get_commission_margin( dominant).iloc[0] for p in param: future_dict[p] = commission[p] future_dict['tick_size'] = rqdatac.instruments( dominant).tick_size() all_futures_info.append(future_dict) with open(os.path.join(d, 'future_info.json'), 'w') as f: json.dump(all_futures_info, f, separators=(',', ':'), indent=2)
def gen_suspended_days(d): from rqdatac.client import get_client stocks = rqdatac.all_instruments('CS').order_book_id.tolist() suspended_days = get_client().execute( 'get_suspended_days', stocks, START_DATE, convert_date_to_date_int(datetime.date.today())) with h5py.File(os.path.join(d, 'suspended_days.h5'), 'w') as h5: for order_book_id, days in suspended_days.items(): h5[order_book_id] = days
def update_bundle(path, create, enable_compression=False, concurrency=1): if create: _DayBarTask = GenerateDayBarTask else: _DayBarTask = UpdateDayBarTask kwargs = {} if enable_compression: kwargs['compression'] = 9 day_bar_args = ( ("stocks.h5", rqdatac.all_instruments('CS').order_book_id.tolist(), STOCK_FIELDS), ("indexes.h5", rqdatac.all_instruments('INDX').order_book_id.tolist(), INDEX_FIELDS), ("futures.h5", rqdatac.all_instruments('Future').order_book_id.tolist(), FUTURES_FIELDS), ("funds.h5", rqdatac.all_instruments('FUND').order_book_id.tolist(), FUND_FIELDS), ) rqdatac.reset() gen_file_funcs = (gen_instruments, gen_trading_dates, gen_dividends, gen_splits, gen_ex_factor, gen_st_days, gen_suspended_days, gen_yield_curve, gen_share_transformation, gen_future_info) with ProgressedProcessPoolExecutor( max_workers=concurrency, initializer=init_rqdatac_with_warnings_catch) as executor: # windows上子进程需要执行rqdatac.init, 其他os则需要执行rqdatac.reset; rqdatac.init包含了rqdatac.reset的功能 for func in gen_file_funcs: executor.submit(GenerateFileTask(func), path) for file, order_book_id, field in day_bar_args: executor.submit(_DayBarTask(order_book_id), os.path.join(path, file), field, **kwargs)
def gen_splits(d): stocks = rqdatac.all_instruments().order_book_id.tolist() split = rqdatac.get_split(stocks) split['split_factor'] = split['split_coefficient_to'] / split[ 'split_coefficient_from'] split = split[['split_factor']] split.reset_index(inplace=True) split.rename(columns={'ex_dividend_date': 'ex_date'}, inplace=True) split['ex_date'] = [convert_date_to_int(d) for d in split['ex_date']] split.set_index(['order_book_id', 'ex_date'], inplace=True) with h5py.File(os.path.join(d, 'split_factor.h5'), 'w') as h5: for order_book_id in split.index.levels[0]: h5[order_book_id] = split.loc[order_book_id].to_records()
def gen_dividends(d): stocks = rqdatac.all_instruments().order_book_id.tolist() dividend = rqdatac.get_dividend(stocks) dividend.reset_index(inplace=True) dividend.rename( columns={'declaration_announcement_date': 'announcement_date'}, inplace=True) for f in ('book_closure_date', 'ex_dividend_date', 'payable_date', 'announcement_date'): dividend[f] = [convert_date_to_date_int(d) for d in dividend[f]] dividend.set_index(['order_book_id', 'book_closure_date'], inplace=True) with h5py.File(os.path.join(d, 'dividends.h5'), 'w') as h5: for order_book_id in dividend.index.levels[0]: h5[order_book_id] = dividend.loc[order_book_id].to_records()
def gen_ex_factor(d): stocks = rqdatac.all_instruments().order_book_id.tolist() ex_factor = rqdatac.get_ex_factor(stocks) ex_factor.reset_index(inplace=True) ex_factor['ex_date'] = [ convert_date_to_int(d) for d in ex_factor['ex_date'] ] ex_factor.rename(columns={'ex_date': 'start_date'}, inplace=True) ex_factor.set_index(['order_book_id', 'start_date'], inplace=True) ex_factor = ex_factor[['ex_cum_factor']] dtype = ex_factor.loc[ex_factor.index.levels[0][0]].to_records().dtype initial = np.empty((1, ), dtype=dtype) initial['start_date'] = 0 initial['ex_cum_factor'] = 1.0 with h5py.File(os.path.join(d, 'ex_cum_factor.h5'), 'w') as h5: for order_book_id in ex_factor.index.levels[0]: h5[order_book_id] = np.concatenate( [initial, ex_factor.loc[order_book_id].to_records()])
def gen_instruments(d): stocks = sorted(list(rqdatac.all_instruments().order_book_id)) instruments = [i.__dict__ for i in rqdatac.instruments(stocks)] with open(os.path.join(d, 'instruments.pk'), 'wb') as out: pickle.dump(instruments, out, protocol=2)
def init_future_info(d): all_futures_info = [] underlying_symbol_list = [] fields = [ 'close_commission_ratio', 'close_commission_today_ratio', 'commission_type', 'open_commission_ratio' ] futures_order_book_id = rqdatac.all_instruments( type='Future')['order_book_id'].unique() for future in futures_order_book_id: future_dict = {} underlying_symbol = re.match(r'^[a-zA-Z]*', future).group() commission = rqdatac.futures.get_commission_margin(future) if not commission.empty: future_dict['order_book_id'] = future commission = commission.iloc[0] for p in fields: future_dict[p] = commission[p] future_dict['tick_size'] = rqdatac.instruments(future).tick_size() elif underlying_symbol not in underlying_symbol_list: if underlying_symbol in {'S', 'TC', 'ER', 'WS', 'WT', 'RO', 'ME'}: continue underlying_symbol_list.append(underlying_symbol) future_dict['underlying_symbol'] = underlying_symbol try: dominant = rqdatac.futures.get_dominant( underlying_symbol).iloc[-1] except AttributeError: # FIXME: why get_dominant return None??? continue commission = rqdatac.futures.get_commission_margin( dominant).iloc[0] for p in fields: future_dict[p] = commission[p] future_dict['tick_size'] = rqdatac.instruments( dominant).tick_size() else: continue all_futures_info.append(future_dict) hard_info = [{ 'underlying_symbol': 'TC', 'close_commission_ratio': 4.0, 'close_commission_today_ratio': 0.0, 'commission_type': "by_volume", 'open_commission_ratio': 4.0, 'tick_size': 0.2 }, { 'underlying_symbol': 'ER', 'close_commission_ratio': 2.5, 'close_commission_today_ratio': 2.5, 'commission_type': "by_volume", 'open_commission_ratio': 2.5, 'tick_size': 1.0 }, { 'underlying_symbol': 'WS', 'close_commission_ratio': 2.5, 'close_commission_today_ratio': 0.0, 'commission_type': "by_volume", 'open_commission_ratio': 2.5, 'tick_size': 1.0 }, { 'underlying_symbol': 'RO', 'close_commission_ratio': 2.5, 'close_commission_today_ratio': 0.0, 'commission_type': "by_volume", 'open_commission_ratio': 2.5, 'tick_size': 2.0 }, { 'underlying_symbol': 'ME', 'close_commission_ratio': 1.4, 'close_commission_today_ratio': 0.0, 'commission_type': "by_volume", 'open_commission_ratio': 1.4, 'tick_size': 1.0 }] all_futures_info += hard_info with open(os.path.join(d, 'future_info.json'), 'w') as f: json.dump(all_futures_info, f, separators=(',', ':'), indent=2)