def gen_future_info(d):
future_info_file = os.path.join(d, 'future_info.json')
if not os.path.exists(future_info_file):
init_future_info(d)
return
with open(future_info_file, 'r') as f:
all_futures_info = json.load(f)
future_list = []
symbol_list = []
param = [
'close_commission_ratio', 'close_commission_today_ratio',
'commission_type', 'open_commission_ratio'
]
for i in all_futures_info:
if i.get('order_book_id'):
future_list.append(i.get('order_book_id'))
else:
symbol_list.append(i.get('underlying_symbol'))
futures_order_book_id = rqdatac.all_instruments(
type='Future')['order_book_id'].unique()
for future in futures_order_book_id:
underlying_symbol = re.match(r'^[a-zA-Z]*', future).group()
if future in future_list:
continue
future_dict = {}
commission = rqdatac.futures.get_commission_margin(future)
if not commission.empty:
future_list.append(future)
future_dict['order_book_id'] = future
commission = commission.iloc[0]
for p in param:
future_dict[p] = commission[p]
future_dict['tick_size'] = rqdatac.instruments(future).tick_size()
elif underlying_symbol in symbol_list \
or underlying_symbol in {'S', 'TC', 'ER', 'WS', 'WT', 'RO', 'ME'}:
continue
else:
symbol_list.append(underlying_symbol)
future_dict['underlying_symbol'] = underlying_symbol
try:
dominant = rqdatac.futures.get_dominant(
underlying_symbol).iloc[-1]
except AttributeError:
# FIXME: why get_dominant return None???
continue
commission = rqdatac.futures.get_commission_margin(
dominant).iloc[0]
for p in param:
future_dict[p] = commission[p]
future_dict['tick_size'] = rqdatac.instruments(
dominant).tick_size()
all_futures_info.append(future_dict)
with open(os.path.join(d, 'future_info.json'), 'w') as f:
json.dump(all_futures_info, f, separators=(',', ':'), indent=2)
def gen_suspended_days(d):
from rqdatac.client import get_client
stocks = rqdatac.all_instruments('CS').order_book_id.tolist()
suspended_days = get_client().execute(
'get_suspended_days', stocks, START_DATE,
convert_date_to_date_int(datetime.date.today()))
with h5py.File(os.path.join(d, 'suspended_days.h5'), 'w') as h5:
for order_book_id, days in suspended_days.items():
h5[order_book_id] = days
def update_bundle(path, create, enable_compression=False, concurrency=1):
if create:
_DayBarTask = GenerateDayBarTask
else:
_DayBarTask = UpdateDayBarTask
kwargs = {}
if enable_compression:
kwargs['compression'] = 9
day_bar_args = (
("stocks.h5", rqdatac.all_instruments('CS').order_book_id.tolist(),
STOCK_FIELDS),
("indexes.h5", rqdatac.all_instruments('INDX').order_book_id.tolist(),
INDEX_FIELDS),
("futures.h5",
rqdatac.all_instruments('Future').order_book_id.tolist(),
FUTURES_FIELDS),
("funds.h5", rqdatac.all_instruments('FUND').order_book_id.tolist(),
FUND_FIELDS),
)
rqdatac.reset()
gen_file_funcs = (gen_instruments, gen_trading_dates, gen_dividends,
gen_splits, gen_ex_factor, gen_st_days,
gen_suspended_days, gen_yield_curve,
gen_share_transformation, gen_future_info)
with ProgressedProcessPoolExecutor(
max_workers=concurrency,
initializer=init_rqdatac_with_warnings_catch) as executor:
# windows上子进程需要执行rqdatac.init, 其他os则需要执行rqdatac.reset; rqdatac.init包含了rqdatac.reset的功能
for func in gen_file_funcs:
executor.submit(GenerateFileTask(func), path)
for file, order_book_id, field in day_bar_args:
executor.submit(_DayBarTask(order_book_id),
os.path.join(path, file), field, **kwargs)
def gen_splits(d):
stocks = rqdatac.all_instruments().order_book_id.tolist()
split = rqdatac.get_split(stocks)
split['split_factor'] = split['split_coefficient_to'] / split[
'split_coefficient_from']
split = split[['split_factor']]
split.reset_index(inplace=True)
split.rename(columns={'ex_dividend_date': 'ex_date'}, inplace=True)
split['ex_date'] = [convert_date_to_int(d) for d in split['ex_date']]
split.set_index(['order_book_id', 'ex_date'], inplace=True)
with h5py.File(os.path.join(d, 'split_factor.h5'), 'w') as h5:
for order_book_id in split.index.levels[0]:
h5[order_book_id] = split.loc[order_book_id].to_records()
def gen_dividends(d):
stocks = rqdatac.all_instruments().order_book_id.tolist()
dividend = rqdatac.get_dividend(stocks)
dividend.reset_index(inplace=True)
dividend.rename(
columns={'declaration_announcement_date': 'announcement_date'},
inplace=True)
for f in ('book_closure_date', 'ex_dividend_date', 'payable_date',
'announcement_date'):
dividend[f] = [convert_date_to_date_int(d) for d in dividend[f]]
dividend.set_index(['order_book_id', 'book_closure_date'], inplace=True)
with h5py.File(os.path.join(d, 'dividends.h5'), 'w') as h5:
for order_book_id in dividend.index.levels[0]:
h5[order_book_id] = dividend.loc[order_book_id].to_records()
def gen_ex_factor(d):
stocks = rqdatac.all_instruments().order_book_id.tolist()
ex_factor = rqdatac.get_ex_factor(stocks)
ex_factor.reset_index(inplace=True)
ex_factor['ex_date'] = [
convert_date_to_int(d) for d in ex_factor['ex_date']
]
ex_factor.rename(columns={'ex_date': 'start_date'}, inplace=True)
ex_factor.set_index(['order_book_id', 'start_date'], inplace=True)
ex_factor = ex_factor[['ex_cum_factor']]
dtype = ex_factor.loc[ex_factor.index.levels[0][0]].to_records().dtype
initial = np.empty((1, ), dtype=dtype)
initial['start_date'] = 0
initial['ex_cum_factor'] = 1.0
with h5py.File(os.path.join(d, 'ex_cum_factor.h5'), 'w') as h5:
for order_book_id in ex_factor.index.levels[0]:
h5[order_book_id] = np.concatenate(
[initial, ex_factor.loc[order_book_id].to_records()])
def gen_instruments(d):
stocks = sorted(list(rqdatac.all_instruments().order_book_id))
instruments = [i.__dict__ for i in rqdatac.instruments(stocks)]
with open(os.path.join(d, 'instruments.pk'), 'wb') as out:
pickle.dump(instruments, out, protocol=2)
def init_future_info(d):
all_futures_info = []
underlying_symbol_list = []
fields = [
'close_commission_ratio', 'close_commission_today_ratio',
'commission_type', 'open_commission_ratio'
]
futures_order_book_id = rqdatac.all_instruments(
type='Future')['order_book_id'].unique()
for future in futures_order_book_id:
future_dict = {}
underlying_symbol = re.match(r'^[a-zA-Z]*', future).group()
commission = rqdatac.futures.get_commission_margin(future)
if not commission.empty:
future_dict['order_book_id'] = future
commission = commission.iloc[0]
for p in fields:
future_dict[p] = commission[p]
future_dict['tick_size'] = rqdatac.instruments(future).tick_size()
elif underlying_symbol not in underlying_symbol_list:
if underlying_symbol in {'S', 'TC', 'ER', 'WS', 'WT', 'RO', 'ME'}:
continue
underlying_symbol_list.append(underlying_symbol)
future_dict['underlying_symbol'] = underlying_symbol
try:
dominant = rqdatac.futures.get_dominant(
underlying_symbol).iloc[-1]
except AttributeError:
# FIXME: why get_dominant return None???
continue
commission = rqdatac.futures.get_commission_margin(
dominant).iloc[0]
for p in fields:
future_dict[p] = commission[p]
future_dict['tick_size'] = rqdatac.instruments(
dominant).tick_size()
else:
continue
all_futures_info.append(future_dict)
hard_info = [{
'underlying_symbol': 'TC',
'close_commission_ratio': 4.0,
'close_commission_today_ratio': 0.0,
'commission_type': "by_volume",
'open_commission_ratio': 4.0,
'tick_size': 0.2
}, {
'underlying_symbol': 'ER',
'close_commission_ratio': 2.5,
'close_commission_today_ratio': 2.5,
'commission_type': "by_volume",
'open_commission_ratio': 2.5,
'tick_size': 1.0
}, {
'underlying_symbol': 'WS',
'close_commission_ratio': 2.5,
'close_commission_today_ratio': 0.0,
'commission_type': "by_volume",
'open_commission_ratio': 2.5,
'tick_size': 1.0
}, {
'underlying_symbol': 'RO',
'close_commission_ratio': 2.5,
'close_commission_today_ratio': 0.0,
'commission_type': "by_volume",
'open_commission_ratio': 2.5,
'tick_size': 2.0
}, {
'underlying_symbol': 'ME',
'close_commission_ratio': 1.4,
'close_commission_today_ratio': 0.0,
'commission_type': "by_volume",
'open_commission_ratio': 1.4,
'tick_size': 1.0
}]
all_futures_info += hard_info
with open(os.path.join(d, 'future_info.json'), 'w') as f:
json.dump(all_futures_info, f, separators=(',', ':'), indent=2)
