def allHisFuture(self, start=None):
columns = [
'maturity_date', 'order_book_id', 'listed_date',
'underlying_symbol'
]
if start is None:
df = rq.all_instruments(type='Future')
else:
df = rq.all_instruments(type='Future', date=public.getDate())
#print(df)
return df[df['maturity_date'] != '0000-00-00'].loc[:, columns]
def dmpCodeListToFile(self,
filename: str,
hasIndex: bool = True,
hasStock: bool = True):
stocks = {"SSE": {}, "SZSE": {}}
#个股列表
if hasStock:
print("Fetching stock list...")
df_stocks = rq.all_instruments(type='CS', market="cn")
for idx, row in df_stocks.iterrows():
rawcode = row["order_book_id"][:6]
exchg = row["exchange"]
if exchg == 'XSHG':
exchg = "SSE"
else:
exchg = "SZSE"
sInfo = dict()
sInfo["exchg"] = exchg
sInfo["code"] = rawcode
sInfo["name"] = row["symbol"]
sInfo["product"] = "STK"
stocks[sInfo["exchg"]][rawcode] = sInfo
if hasIndex:
#上证指数列表
print("Fetching index list...")
df_stocks = rq.all_instruments(type='INDX', market="cn")
for idx, row in df_stocks.iterrows():
rawcode = row["order_book_id"][:6]
exchg = row["exchange"]
if exchg == 'XSHG':
exchg = "SSE"
else:
exchg = "SZSE"
sInfo = dict()
sInfo["exchg"] = exchg
sInfo["code"] = rawcode
sInfo["name"] = row["symbol"]
sInfo["product"] = "IDX"
stocks[sInfo["exchg"]][rawcode] = sInfo
print("Writing code list into file %s..." % (filename))
f = open(filename, 'w')
f.write(
json.dumps(stocks, sort_keys=True, indent=4, ensure_ascii=False))
f.close()
def update_option(message=None):
#Replace all contract infomation
SQL = create_engine('postgresql://*****:*****@#[email protected]:5432/option')
all_instruments = rq.all_instruments(type="Option")
all_instruments.to_sql("contract", SQL, if_exists='replace')
for id, group in all_instruments.groupby(["underlying_order_book_id"]):
#Update option contracts
contracts = options.get_contracts(id)
for contract in contracts:
table = "daily_%s" % contract
if is_table_exist(table, SQL):
order_book_id_df = all_instruments[all_instruments["order_book_id"] == contract]
maturity_date_string = order_book_id_df["maturity_date"].values[0]
maturity_date_datetime = datetime.datetime(year=int(maturity_date_string[0:4]),
month=int(maturity_date_string[5:7]),
day=int(maturity_date_string[8:10]))
if maturity_date_datetime > TODAY:
if get_started_date(table, SQL):
start_date_string = get_started_date(table, SQL)
df = rq.get_price(contract, start_date=start_date_string, end_date=TODAY_STR, frequency='1d')
write(df, table, SQL, if_exists='append')
print(u"已更新期权：%s 日线行情,开始时间: %s, 结束时间: %s" % (contract, start_date_string, TODAY_STR))
else:
df = rq.get_price(contract, start_date=START_DATE_STR, end_date=TODAY_STR, frequency='1d')
write(df, table, SQL, if_exists='replace')
print(u"已更新期权：%s 日线行情,开始时间: %s, 结束时间: %s" % (contract, START_DATE_STR, TODAY_STR))
def crawl(self, begin_date=None, end_date=None):
if begin_date is None and end_date is None:
return
# 区间交易日
dates = rqd.get_trading_dates(begin_date, end_date)
for date in dates:
# 当天所有股票
stocks = rqd.all_instruments(type='CS',
date=date)['order_book_id'].tolist()
update_requests = []
for i in rqd.instruments(stocks):
doc = i.__dict__
# 股票信息与时间无关
update_requests.append(
UpdateOne({'code': doc['order_book_id']}, {'$set': doc},
upsert=True))
if len(update_requests) > 0:
# bulk_write 批量写入
# 写入daily数据集合(表) , 不按顺序ordered = False
update_result = DB_CONN['basic'].bulk_write(update_requests,
ordered=False)
print('保存-%s-%s数据 , 插入:%4d , 更新:%4d' %
(date, 'basic', update_result.upserted_count,
update_result.modified_count),
flush=True)
def getallInstrments():
all = rq.all_instruments(type='Future', country='cn')
tmp_date = datetime.now() - timedelta(days=20 * 365)
all = all.query("maturity_date <= '{0}'".format(tmp_date))
return (all)
def update_basedata():
basedata = arctic['basedata']
stocks = rq.all_instruments('CS')
idx = rq.all_instruments('INDX')
try:
updated = basedata.list_versions()[0]['date'].date()
if updated == date.today():
print('Already updated today')
return
except:
pass
documents = {'stocks': stocks, 'indexes': idx}
for label, data in documents.items():
basedata.write(label, data)
def gen_suspended_days(d):
from rqdatac.client import get_client
stocks = rqdatac.all_instruments('CS').order_book_id.tolist()
suspended_days = get_client().execute(
'get_suspended_days', stocks, START_DATE,
convert_date_to_date_int(datetime.date.today()))
with h5py.File(os.path.join(d, 'suspended_days.h5'), 'w') as h5:
for order_book_id, days in suspended_days.items():
h5[order_book_id] = days
def get_implicit_factor_return(date):
latest_trading_date = str(
rqdatac.get_previous_trading_date(
datetime.strptime(date, "%Y-%m-%d") + timedelta(days=1)))
previous_trading_date = str(
rqdatac.get_previous_trading_date(latest_trading_date))
# 取前一交易日全市场已经上市的股票，保证日收益率计算
stock_list = rqdatac.all_instruments(
type='CS', date=previous_trading_date)['order_book_id'].tolist()
# 剔除上市不满21天的股票
trading_date_21_days_before = str(
rqdatac.get_previous_trading_date(latest_trading_date,
country='cn',
n=21))
stock_list = [
i for i in stock_list
if rqdatac.instruments(i).listed_date <= trading_date_21_days_before
]
# 剔除ST股
is_st_df = rqdatac.is_st_stock(stock_list,
start_date=previous_trading_date,
end_date=previous_trading_date)
is_st_df.index = is_st_df.index.astype(str)
stock_list = is_st_df.loc[previous_trading_date][
is_st_df.loc[previous_trading_date].values == False].index.tolist()
# 剔除停牌股
trading_volume = rqdatac.get_price(stock_list,
start_date=previous_trading_date,
end_date=previous_trading_date,
frequency='1d',
fields='volume',
country='cn')
stock_list = trading_volume.loc[previous_trading_date][
trading_volume.loc[previous_trading_date].values > 0].index.tolist()
# 计算全市场前一交易日的行业暴露度
factor_exposure = get_exposure(stock_list, str(previous_trading_date))
# 根据上述四类暴露度计算因子收益率
factor_returns = factor_return_estimation(latest_trading_date,
factor_exposure)
return factor_returns
def gen_future_info(d):
future_info_file = os.path.join(d, 'future_info.json')
if not os.path.exists(future_info_file):
init_future_info(d)
return
with open(future_info_file, 'r') as f:
all_futures_info = json.load(f)
future_list = []
symbol_list = []
param = [
'close_commission_ratio', 'close_commission_today_ratio',
'commission_type', 'open_commission_ratio'
]
for i in all_futures_info:
if i.get('order_book_id'):
future_list.append(i.get('order_book_id'))
else:
symbol_list.append(i.get('underlying_symbol'))
futures_order_book_id = rqdatac.all_instruments(
type='Future')['order_book_id'].unique()
for future in futures_order_book_id:
underlying_symbol = re.match(r'^[a-zA-Z]*', future).group()
if future in future_list:
continue
future_dict = {}
commission = rqdatac.futures.get_commission_margin(future)
if not commission.empty:
future_list.append(future)
future_dict['order_book_id'] = future
commission = commission.iloc[0]
for p in param:
future_dict[p] = commission[p]
future_dict['tick_size'] = rqdatac.instruments(future).tick_size()
elif underlying_symbol in symbol_list \
or underlying_symbol in {'S', 'TC', 'ER', 'WS', 'WT', 'RO', 'ME'}:
continue
else:
symbol_list.append(underlying_symbol)
future_dict['underlying_symbol'] = underlying_symbol
dominant = rqdatac.futures.get_dominant(underlying_symbol).iloc[-1]
commission = rqdatac.futures.get_commission_margin(
dominant).iloc[0]
for p in param:
future_dict[p] = commission[p]
future_dict['tick_size'] = rqdatac.instruments(
dominant).tick_size()
all_futures_info.append(future_dict)
with open(os.path.join(d, 'future_info.json'), 'w') as f:
json.dump(all_futures_info, f, separators=(',', ':'), indent=2)
def baseUpdate(self):
df = rq.all_instruments(type='Future', date=public.getDate())
u = df["underlying_symbol"].unique()
FT = future_baseInfo()
# 已有code及结束时间
exists = FT.exists()
exCodes = [c[0] for c in exists]
endMap = {}
for c in exists:
endMap[c[0]] = c[1]
docs = []
mu = self.getMain(u)
i = 0
for d in u:
n = df[df["underlying_symbol"] ==
d].loc[:, FT.keylist[2:]].values[-1].tolist()
doc = FT.set([d, 0] + n)
detais = rq.instruments(mu[i])
# tick_size
doc["tick_size"] = detais.tick_size()
doc['contract_multiplier'] = detais.contract_multiplier
doc['margin_rate'] = detais.margin_rate
# 结束时间
hh = detais.trading_hours.split(',')
hs = hh[0].split('-')
if hs[0][0:2] != '09':
doc["nightEnd"] = hs[1]
else:
doc["nightEnd"] = hh[-1].split('-')[1]
if d not in exCodes:
doc['product'] = 'Commodity'
docs.append(doc)
else:
# 更新结束时间
if doc["nightEnd"] != endMap[d][0:len(doc["nightEnd"])]:
print(d, endMap[d][0:len(doc["nightEnd"])],
doc["nightEnd"])
FT.setMulti(doc)
i += 1
if len(docs) > 0:
FT.insertAll(docs)
logger.info(('future base update finished, ', docs))
def gen_dividends(d):
stocks = rqdatac.all_instruments().order_book_id.tolist()
dividend = rqdatac.get_dividend(stocks)
dividend.reset_index(inplace=True)
dividend.rename(
columns={'declaration_announcement_date': 'announcement_date'},
inplace=True)
for f in ('book_closure_date', 'ex_dividend_date', 'payable_date',
'announcement_date'):
dividend[f] = [convert_date_to_date_int(d) for d in dividend[f]]
dividend.set_index(['order_book_id', 'book_closure_date'], inplace=True)
with h5py.File(os.path.join(d, 'dividends.h5'), 'w') as h5:
for order_book_id in dividend.index.levels[0]:
h5[order_book_id] = dividend.loc[order_book_id].to_records()
def gen_splits(d):
stocks = rqdatac.all_instruments().order_book_id.tolist()
split = rqdatac.get_split(stocks)
split['split_factor'] = split['split_coefficient_to'] / split[
'split_coefficient_from']
split = split[['split_factor']]
split.reset_index(inplace=True)
split.rename(columns={'ex_dividend_date': 'ex_date'}, inplace=True)
split['ex_date'] = [convert_date_to_int(d) for d in split['ex_date']]
split.set_index(['order_book_id', 'ex_date'], inplace=True)
with h5py.File(os.path.join(d, 'split_factor.h5'), 'w') as h5:
for order_book_id in split.index.levels[0]:
h5[order_book_id] = split.loc[order_book_id].to_records()
def get_implicit_factor_return(date):
latest_trading_date = str(
rqdatac.get_previous_trading_date(
datetime.strptime(date, "%Y-%m-%d") + timedelta(days=1)))
previous_trading_date = str(
rqdatac.get_previous_trading_date(latest_trading_date))
# 取前一交易日全市场已经上市的股票，保证日收益率计算
stock_list = rqdatac.all_instruments(
type='CS',
date=previous_trading_date)['order_book_id'].values.tolist()
# 计算全市场前一交易日的行业暴露度
rq_exposure, rq_style_barra_industry, barra_exposure, barra_style_rq_industry_exposure = get_exposure(
stock_list, previous_trading_date)
# 根据上述四类暴露度计算因子收益率
shenwan_industry_name = ['农林牧渔', '采掘', '化工', '钢铁', '有色金属', '电子', '家用电器', '食品饮料', '纺织服装', '轻工制造',\
'医药生物', '公用事业', '交通运输', '房地产', '商业贸易', '休闲服务','综合', '建筑材料', '建筑装饰', '电气设备',\
'国防军工', '计算机', '传媒', '通信', '银行', '非银金融', '汽车', '机械设备']
industry_factors = [
'CNE5S_ENERGY', 'CNE5S_CHEM', 'CNE5S_CONMAT', 'CNE5S_MTLMIN',
'CNE5S_MATERIAL', 'CNE5S_AERODEF', 'CNE5S_BLDPROD', 'CNE5S_CNSTENG',
'CNE5S_ELECEQP', 'CNE5S_INDCONG', 'CNE5S_MACH', 'CNE5S_TRDDIST',
'CNE5S_COMSERV', 'CNE5S_AIRLINE', 'CNE5S_MARINE', 'CNE5S_RDRLTRAN',
'CNE5S_AUTO', 'CNE5S_HOUSEDUR', 'CNE5S_LEISLUX', 'CNE5S_CONSSERV',
'CNE5S_MEDIA', 'CNE5S_RETAIL', 'CNE5S_PERSPRD', 'CNE5S_BEV',
'CNE5S_FOODPROD', 'CNE5S_HEALTH', 'CNE5S_BANKS', 'CNE5S_DVFININS',
'CNE5S_REALEST', 'CNE5S_SOFTWARE', 'CNE5S_HDWRSEMI', 'CNE5S_UTILITIE'
]
rq_factor_returns = factor_return_estimation(date, rq_exposure,
shenwan_industry_name)
rq_style_barra_industry_factor_returns = factor_return_estimation(
date, rq_style_barra_industry, industry_factors)
barra_factor_returns = factor_return_estimation(date, barra_exposure,
industry_factors)
barra_style_rq_industry_factor_returns = factor_return_estimation(
date, barra_style_rq_industry_exposure, shenwan_industry_name)
return rq_factor_returns, rq_style_barra_industry_factor_returns, barra_factor_returns, barra_style_rq_industry_factor_returns
def update_bundle(path, create, enable_compression=False, concurrency=1):
if create:
_DayBarTask = GenerateDayBarTask
else:
_DayBarTask = UpdateDayBarTask
kwargs = {}
if enable_compression:
kwargs['compression'] = 9
day_bar_args = (
("stocks.h5", rqdatac.all_instruments('CS').order_book_id.tolist(),
STOCK_FIELDS),
("indexes.h5", rqdatac.all_instruments('INDX').order_book_id.tolist(),
INDEX_FIELDS),
("futures.h5",
rqdatac.all_instruments('Future').order_book_id.tolist(),
FUTURES_FIELDS),
("funds.h5", rqdatac.all_instruments('FUND').order_book_id.tolist(),
FUND_FIELDS),
)
gen_file_funcs = (gen_instruments, gen_trading_dates, gen_dividends,
gen_splits, gen_ex_factor, gen_st_days,
gen_suspended_days, gen_yield_curve,
gen_share_transformation, gen_future_info)
with ProgressedProcessPoolExecutor(
max_workers=concurrency,
initializer=init_rqdatac_with_warnings_catch) as executor:
# windows上子进程需要执行rqdatac.init, 其他os则需要执行rqdatac.reset; rqdatac.init包含了rqdatac.reset的功能
for func in gen_file_funcs:
executor.submit(GenerateFileTask(func), path)
for file, order_book_id, field in day_bar_args:
executor.submit(_DayBarTask(order_book_id),
os.path.join(path, file), field, **kwargs)
def pure_factor_return(date):
latest_trading_date = str(
rqdatac.get_previous_trading_date(
datetime.strptime(date, "%Y-%m-%d") + timedelta(days=1)))
previous_trading_date = str(
rqdatac.get_previous_trading_date(latest_trading_date))
# 取前一交易日全市场已经上市的股票，保证日收益率计算
stock_list = rqdatac.all_instruments(
type='CS',
date=previous_trading_date)['order_book_id'].values.tolist()
# 计算全市场前一交易日的行业暴露度
factor_exposure = get_exposure(stock_list,
previous_trading_date,
industry_classification='GICS')
# 计算因子收益
style_factors = [
'CNE5S_BETA', 'CNE5S_MOMENTUM', 'CNE5S_SIZE', 'CNE5S_EARNYILD',
'CNE5S_RESVOL', 'CNE5S_GROWTH', 'CNE5S_BTOP', 'CNE5S_LEVERAGE',
'CNE5S_LIQUIDTY', 'CNE5S_SIZENL'
]
industry_factors = ['CNE5S_ENERGY', 'CNE5S_CHEM', 'CNE5S_CONMAT', 'CNE5S_MTLMIN', 'CNE5S_MATERIAL', 'CNE5S_AERODEF',\
'CNE5S_BLDPROD', 'CNE5S_CNSTENG', 'CNE5S_ELECEQP', 'CNE5S_INDCONG', 'CNE5S_MACH', 'CNE5S_TRDDIST',\
'CNE5S_COMSERV', 'CNE5S_AIRLINE', 'CNE5S_MARINE', 'CNE5S_RDRLTRAN', 'CNE5S_AUTO', 'CNE5S_HOUSEDUR',\
'CNE5S_LEISLUX', 'CNE5S_CONSSERV', 'CNE5S_MEDIA', 'CNE5S_RETAIL', 'CNE5S_PERSPRD', 'CNE5S_BEV',\
'CNE5S_FOODPROD', 'CNE5S_HEALTH', 'CNE5S_BANKS', 'CNE5S_DVFININS', 'CNE5S_REALEST', 'CNE5S_SOFTWARE',\
'CNE5S_HDWRSEMI', 'CNE5S_UTILITIE']
# 重新排序，保证先风格因子，后行业因子
reorganized_factor_exposure = pd.concat([
factor_exposure[style_factors], factor_exposure[industry_factors],
factor_exposure['CNE5S_COUNTRY']
],
axis=1)
factor_returns = factor_return_estimation(
stock_list, date=date, factor_exposure=reorganized_factor_exposure)
return factor_returns
def compute_one_day(self, date):
# 用不到,只是为了练习,实际取米筐数据存
# dm = DataModule()
# 某天所有股票数据
# df_daily = dm.get_all_price_one_day(date)
stocklist = rqd.all_instruments('CS', date)['order_book_id'].tolist()
pe = rqd.get_factor(stocklist, self.name, date, date)
factors = []
for code in pe.index:
factors.append({
'date': date,
'code': code,
self.name: round(pe[code], 2)
})
return factors
def get_explicit_factor_returns(date):
"""
:param date:日期
:return: pandas.Series
"""
previous_trading_date = rqdatac.get_previous_trading_date(date)
all_a_stocks = rqdatac.all_instruments(type="CS",date=previous_trading_date).order_book_id.tolist()
filtered_stocks = noisy_stocks_filter(all_a_stocks,previous_trading_date)
# print(all_a_stocks,previous_trading_date)
factor_exposures = rqdatac.get_style_factor_exposure(all_a_stocks, previous_trading_date, previous_trading_date, "all").sort_index()
factor_exposures.index=factor_exposures.index.droplevel(1)
closePrice = rqdatac.get_price(all_a_stocks, rqdatac.get_previous_trading_date(previous_trading_date),
previous_trading_date, fields="close")
priceChange = closePrice.pct_change().iloc[-1]
index_mapping = {"csi_300":'000300.XSHG',"csi_500":"000905.XSHG","csi_800":"000906.XSHG"}
all_stocks = {index:rqdatac.index_components(index_mapping.get(index),date=previous_trading_date) for index in index_mapping}
all_stocks['whole_market'] = filtered_stocks
def _calc_explicitReturns_with_stocksList(stocksList):
# 根据股票池计算收益率
_sizeBeta = factor_exposures[['size','beta']].loc[stocksList]
_quantileGroup = _sizeBeta.apply(lambda x:pd.cut(x,bins=3,labels=False)+1).reset_index()
_quantileStocks = _quantileGroup.groupby(['size','beta']).apply(lambda x:x.index.tolist())
market_neutralize_stocks = _quantileStocks.apply(
lambda x: pd.Series(stocksList).loc[x].values.tolist()).values.tolist()
return factor_exposures.loc[stocksList].apply(lambda x,y=market_neutralize_stocks:_calc_single_explicit_returns(x,y))
def _calc_single_explicit_returns(_factor_exposure,market_neutralize_stocks):
# 计算单一因子收益率
def _deuce(series):
median = series.median()
return [series[series<=median].index.tolist(),series[series>median].index.tolist()]
deuceResults = np.array([_deuce(_factor_exposure[neutralized_stks]) for neutralized_stks in market_neutralize_stocks]).flatten()
short_stocksList = list(reduce(lambda x,y:set(x)|set(y),np.array([s for i,s in enumerate(deuceResults) if i%2==0])))
long_stockList = list(reduce(lambda x,y:set(x)|set(y),np.array([s for i,s in enumerate(deuceResults) if i%2==1])))
return priceChange[long_stockList].mean() - priceChange[short_stocksList].mean()
results = {key: _calc_explicitReturns_with_stocksList(all_stocks.get(key)) for key in all_stocks}
return pd.DataFrame(results)[['whole_market','csi_300','csi_500','csi_800']]
def update_future(message=None):
all_instruments = rq.all_instruments(type="Future")
sql.insert(all_instruments, "future/contract", "replace")
futures = ["CU", "M", "SR"]
for i in futures:
data = all_instruments[all_instruments.underlying_symbol == i]
data = data[data.symbol.str.contains("主力连续")]
ids = data.order_book_id
for id in ids:
if message:
message.append(u"正在更新期货合约：%s 主力连续日线数据" % id)
message.update()
else:
print(u"正在更新期货合约：%s 主力连续日线数据" % id)
table = "future/contracts/%s" % id
if sql.is_table(table):
if is_future_contract_need_update(table):
start_date_string = get_started_date(table)
df = rq.get_price(id,
start_date=start_date_string,
end_date=TODAY_STR,
frequency='1d')
if not sql.insert(
df, table, "append", index=True,
index_label="date"):
df = rq.get_price(id,
start_date=START_DATE_STR,
end_date=TODAY_STR,
frequency='1d')
sql.insert(df,
table,
"replace",
index=True,
index_label="date")
else:
df = rq.get_price(id,
start_date=START_DATE_STR,
end_date=TODAY_STR,
frequency='1d')
sql.insert(df,
table,
"replace",
index=True,
index_label="date")
return
def get_basic_information(_date) -> pd.DataFrame:
"""
:return: pandas dataframe, index[ nan ]: [[], [], .... ]
"""
try:
_origin_data = rqdatac.all_instruments(type='Option',
date=_date)[_REQUEST_ATTR]
except ConnectionAbortedError:
raise ConnectionAbortedError('Connection error happens')
_origin_data['de_listed_date'] = _origin_data['de_listed_date'].apply(
lambda x: dt.datetime.strptime(x, '%Y-%m-%d'))
_origin_data = _origin_data[_origin_data['de_listed_date'] > _date]
_origin_data['listed_date'] = _origin_data['listed_date'].apply(
lambda x: dt.datetime.strptime(x, '%Y-%m-%d'))
_origin_data = _origin_data[
_origin_data['underlying_order_book_id'].str.contains(_FILTER_MAP)]
return _origin_data
def get_trading_dates_all_option(end_date, start_date=None) -> list:
"""
:param start_date: define start date yourself
:param end_date: datetime, the end date
:return: list if trading dates
"""
if start_date is None:
_ori_date = rqdatac.all_instruments(type='Option')
all_listed_date = _ori_date['listed_date'].apply(
lambda x: dt.datetime.strptime(x, '%Y-%m-%d')).tolist()
earliest_list_date = min(all_listed_date)
else:
earliest_list_date = start_date
if end_date < earliest_list_date:
return []
# get trading date
trading_dates = rqdatac.get_trading_dates(earliest_list_date, end_date)
return trading_dates
def gen_ex_factor(d):
stocks = rqdatac.all_instruments().order_book_id.tolist()
ex_factor = rqdatac.get_ex_factor(stocks)
ex_factor.reset_index(inplace=True)
ex_factor['ex_date'] = [
convert_date_to_int(d) for d in ex_factor['ex_date']
]
ex_factor.rename(columns={'ex_date': 'start_date'}, inplace=True)
ex_factor.set_index(['order_book_id', 'start_date'], inplace=True)
ex_factor = ex_factor[['ex_cum_factor']]
dtype = ex_factor.loc[ex_factor.index.levels[0][0]].to_records().dtype
initial = np.empty((1, ), dtype=dtype)
initial['start_date'] = 0
initial['ex_cum_factor'] = 1.0
with h5py.File(os.path.join(d, 'ex_cum_factor.h5'), 'w') as h5:
for order_book_id in ex_factor.index.levels[0]:
h5[order_book_id] = np.concatenate(
[initial, ex_factor.loc[order_book_id].to_records()])
def update_stock():
#SQL = create_engine('postgresql://*****:*****@#[email protected]:5432/stock')
SQL = create_engine('postgresql://*****:*****@#[email protected]:5432/stock')
all_instruments = rq.all_instruments(type="CS")
all_instruments.to_sql("contract", SQL, if_exists='replace')
contracts_df = pd.read_sql_query('select * from "contract"', con=SQL)
ids = contracts_df["order_book_id"]
for id in ids:
table = "daily_%s" % id
if is_table_exist(table, SQL):
start_date_string = get_started_date(table, SQL)
if start_date_string:
df = rq.get_price(id, start_date=start_date_string, end_date=TODAY_STR, frequency='1d')
write(df, table, SQL, if_exists='append')
print(u"已更新股票：%s 日线行情,开始时间: %s, 结束时间: %s" % (id, start_date_string, TODAY_STR))
else:
df = rq.get_price(id, start_date=START_DATE_STR, end_date=TODAY_STR, frequency='1d')
write(df, table, SQL, if_exists='replace')
print(u"已更新股票：%s 日线行情,开始时间: %s, 结束时间: %s" % (id, START_DATE_STR, TODAY_STR))
return
def get_implicit_factor_return(date):
latest_trading_date = str(rqdatac.get_previous_trading_date(datetime.strptime(date, "%Y-%m-%d") + timedelta(days=1)))
previous_trading_date = str(rqdatac.get_previous_trading_date(latest_trading_date))
# 取前一交易日全市场已经上市的股票，保证日收益率计算
stock_list = rqdatac.all_instruments(type='CS', date=previous_trading_date)['order_book_id'].values.tolist()
trading_volume = rqdatac.get_price(stock_list, start_date=date, end_date=date, frequency='1d', fields='volume',country='cn')
stock_list = trading_volume.loc[date][trading_volume.loc[date].values > 0].index.tolist()
# 计算全市场前一交易日的行业暴露度
factor_exposure = get_exposure(stock_list,str(previous_trading_date))
# 根据上述四类暴露度计算因子收益率
factor_returns = factor_return_estimation(date, factor_exposure)
return factor_returns
def update_future():
SQL = create_engine('postgresql://*****:*****@#[email protected]:5432/future')
all_instruments = rq.all_instruments(type="Future")
all_instruments.to_sql("contract", SQL, if_exists='replace')
futures = ["CU", "M", "SR"]
for i in futures:
data = all_instruments[all_instruments.underlying_symbol == i]
data = data[data.symbol.str.contains("主力连续")]
ids = data.order_book_id
for id in ids:
table = "daily_%s" % id
if is_table_exist(table, SQL):
start_date_string = get_started_date(table, SQL)
if start_date_string:
df = rq.get_price(id, start_date=start_date_string, end_date=TODAY_STR, frequency='1d')
write(df, table, SQL, if_exists='append')
print(u"已更新期货合约：%s 主力连续日线数据, 开始时间: %s, 结束时间: %s" % (id, start_date_string, TODAY_STR))
else:
df = rq.get_price(id, start_date=START_DATE_STR, end_date=TODAY_STR, frequency='1d')
write(df, table, SQL, if_exists='replace')
print(u"已更新期货合约：%s 主力连续日线数据, 开始时间: %s, 结束时间: %s" % (id, START_DATE_STR, TODAY_STR))
return
def get_momentum_and_res_vol(date):
latest_trading_date = rqdatac.get_previous_trading_date(datetime.strptime(date, "%Y-%m-%d") + timedelta(days=1))
trading_date_252_before = rqdatac.get_trading_dates(latest_trading_date - timedelta(days=500), latest_trading_date, country='cn')[-252]
stock_list = rqdatac.all_instruments(type='CS', date=latest_trading_date)['order_book_id'].values.tolist()
### 获取因子计算共用的行情数据和财务数据
recent_report_type, annual_report_type, market_cap_on_current_day, \
stock_excess_return, market_portfolio_excess_return, recent_five_annual_shares, \
last_reported_non_current_liabilities, last_reported_preferred_stock = get_financial_and_market_data(stock_list,latest_trading_date,trading_date_252_before)
# momentum和residual volatility计算
market_portfolio_beta, market_portfolio_beta_exposure = get_market_portfolio_beta(stock_excess_return,market_portfolio_excess_return,market_cap_on_current_day)
momentum = get_momentum(stock_list, latest_trading_date, market_cap_on_current_day)
daily_standard_deviation, cumulative_range, historical_sigma, residual_volatility = get_residual_volatility(
stock_list, latest_trading_date, stock_excess_return, market_portfolio_excess_return, market_cap_on_current_day,
market_portfolio_beta_exposure, market_portfolio_beta)
style_factors_exposure = pd.concat([momentum, residual_volatility], axis=1)
style_factors_exposure.columns = ['momentum', 'residual_volatility']
atomic_descriptors_exposure = pd.concat(
[daily_standard_deviation, cumulative_range, historical_sigma], axis=1)
atomic_descriptors_exposure.columns = ['daily_standard_deviation', 'cumulative_range', 'historical_sigma']
# 提取财务数据的时候，会提取当前全市场股票的数据，因此 dataframe 中可能包含计算日期未上市的股票，需要对 style_factors_exposure 取子集
atomic_descriptors_exposure = atomic_descriptors_exposure.loc[stock_list]
style_factors_exposure = style_factors_exposure.loc[stock_list]
# 用回归方法处理细分因子的缺失值
imputed_atomic_descriptors = pd.DataFrame()
for atomic_descriptor in atomic_descriptors_exposure.columns:
imputed_atomic_descriptors[atomic_descriptor] = individual_factor_imputation(stock_list, atomic_descriptors_exposure[atomic_descriptor], market_cap_on_current_day,latest_trading_date.strftime('%Y-%m-%d'))
# 用回归方法处理风格因子暴露度的缺失值
imputed_style_factors_exposure = style_factors_imputation(style_factors_exposure, market_cap_on_current_day,latest_trading_date.strftime('%Y-%m-%d'))
# 若经过缺失值处理后因子暴露度依旧存在缺失值，使用全市场股票进行回归，填补缺失值
if imputed_style_factors_exposure.isnull().sum().sum() > 0:
imputed_style_factors_exposure = factor_imputation(market_cap_on_current_day,imputed_style_factors_exposure)
if imputed_atomic_descriptors.isnull().sum().sum() > 0:
imputed_atomic_descriptors = factor_imputation(market_cap_on_current_day,imputed_atomic_descriptors)
return imputed_atomic_descriptors, imputed_style_factors_exposure
def get_contracts(self):
ticker_df = rq.all_instruments(type='Future')
ticker_list = [(tp, self._contracts_(ticker_df, tp))
for tp in self.types]
self.ticker_dict = {id: (tp, df) for tp, (id, df) in ticker_list}
return self.ticker_dict
industry_exposure = get_shenwan_industry_exposure(stock_list, date)
style_exposure = rqdatac.get_style_factor_exposure(stock_list, date, date, factors = 'all')
style_exposure.index = style_exposure.index.droplevel('date')
factor_exposure = pd.concat([style_exposure,industry_exposure],axis=1)
factor_exposure['comovement'] = 1
return factor_exposure
date = '2018-07-12'
stock_list = rqdatac.all_instruments(type='CS',country='cn',date=date).order_book_id.tolist()
factor_exposure = get_factor_exposure(stock_list,date)
style_exposure = rqdatac.get_style_factor_exposure(stock_list, date, date, factors=['beta'])
style_exposure.index = style_exposure.index.droplevel('date')
liquidity_exposure = rqdatac.get_factor_exposure(stock_list,date,date,factors=['beta'])
liquidity_exposure.index = liquidity_exposure.index.droplevel('date')
data = pd.concat([factor_exposure['beta'],style_exposure,liquidity_exposure],axis=1)
data.columns=['源代码计算beta暴露度结果','get_style_factor_exposure API','get_factor_exposure API']
def import_future_info(chain_param=None):
"""
更新期货合约列表信息
:param chain_param: 在celery 中將前面結果做爲參數傳給後面的任務
:return:
"""
table_name = "rqdatac_future_info"
has_table = engine_md.has_table(table_name)
logger.info("更新 %s 开始", table_name)
# 获取已存在合约列表
if has_table:
sql_str = 'select max(listed_date) from {table_name}'.format(
table_name=table_name)
with with_db_session(engine_md) as session:
listed_date = session.scalar(sql_str)
if listed_date is None:
has_table = False
ndays_per_update = 60
# 获取接口参数以及参数列表
col_name_param_list = [
("order_book_id", String(20)),
# 期货代码，期货的独特的标识符（郑商所期货合约数字部分进行了补齐。例如原有代码'ZC609'补齐之后变为'ZC1609'）。主力连续合约 UnderlyingSymbol+88，例如'IF88' ；指数连续合约命名规则为 UnderlyingSymbol+99
("underlying_symbol", String(20)), # 合约标的名称，例如 IF1005 的合约标的名称为'IF'
("market_tplus", TINYINT), # 交易制度。'0'表示 T+0，'1'表示 T+1，往后顺推
("symbol", String(100)), # 期货的简称，例如'沪深 1005'
("margin_rate", DOUBLE), # 期货合约的最低保证金率
("maturity_date", Date), # 期货到期日。主力连续合约与指数连续合约都为'0000-00-00'
("type", String(20)), # 合约类型，'Future'
("trading_code", String(20)), #
(
"exchange", String(10)
), # 交易所，'DCE' - 大连商品交易所, 'SHFE' - 上海期货交易所，'CFFEX' - 中国金融期货交易所, 'CZCE'- 郑州商品交易所
("product",
String(20)), # 合约种类，'Commodity'-商品期货，'Index'-股指期货，'Government'-国债期货
("contract_multiplier", SMALLINT), # 合约乘数，例如沪深 300 股指期货的乘数为 300.0
("round_lot", TINYINT), # 期货全部为 1.0
("trading_hours", String(100)), # 合约交易时间
("listed_date", Date), # 期货的上市日期。主力连续合约与指数连续合约都为'0000-00-00'
("industry_name", String(50)),
("de_listed_date", Date), # 目测与 maturity_date 相同
("underlying_order_book_id",
String(20)), # 合约标的代码，目前除股指期货(IH, IF, IC)之外的期货合约，这一字段全部为'null'
]
dtype = {key: val for key, val in col_name_param_list}
if not has_table:
instrument_info_df = rqdatac.all_instruments(type='Future',
market='cn')
else:
date_yestoday = date.today() - timedelta(days=1)
ndays_per_update = 60
instrument_info_df = None
for param in get_date_iter(listed_date, date_yestoday,
ndays_per_update):
if instrument_info_df is None:
instrument_info_df = rqdatac.all_instruments(type='Future',
market='cn',
date=param)
else:
instrument_info_df_tmp = rqdatac.all_instruments(type='Future',
market='cn',
date=param)
instrument_info_df.append(instrument_info_df_tmp)
instrument_info_df.drop_duplicates(inplace=True)
instrument_info_df.loc[instrument_info_df['underlying_order_book_id'] ==
'null', 'underlying_order_book_id'] = None
instrument_info_df.rename(
columns={c: str.lower(c)
for c in instrument_info_df.columns},
inplace=True)
data_count = bunch_insert_on_duplicate_update(instrument_info_df,
table_name,
engine_md,
dtype=dtype)
logging.info("更新 %s 结束 %d 条信息被更新", table_name, data_count)
if not has_table and engine_md.has_table(table_name):
alter_table_2_myisam(engine_md, [table_name])
build_primary_key([table_name])
logger.info("更新 wind_future_info 结束 %d 条记录被更新", data_count)
import rqdatac as rq
"""
合约贴水率分析
1 到期日前推，贴水率与期限的关系
2 不同年份同一月份合约的贴水率，主要考察季月合约贴水率的变化，观察离到期日较远时季月合约的贴水率变化
观察9、12月合约的分红季的贴水
"""
if __name__ == '__main__':
rq.init("ricequant", "8ricequant8", ('10.29.135.119', 16010))
# startDate = '2021-01-01'
# endDate = '2021-03-12'
underlyingSymbol = 'IC'
allInstruments = rq.all_instruments(type='Future')
underlyingInstruments = allInstruments[allInstruments['underlying_symbol']
== underlyingSymbol]
underlyingInstruments = underlyingInstruments[~(
underlyingInstruments['listed_date'] == '0000-00-00')]
underlyingInstruments = underlyingInstruments.sort_values(by='listed_date',
ascending=True)
indexCode = underlyingInstruments['underlying_order_book_id'].values[0]
# dateList = rq.get_trading_dates(start_date=startDate, end_date=endDate)
#
# contractData = pd.DataFrame(columns=['one', 'two', 'three', 'four'])
# discountData = pd.DataFrame(columns=['one', 'two', 'three', 'four'])
# for dateInd in dateList:
# # dateInd = dateList[0]
def get_universe(date):
return (rqd.all_instruments(
type='CS',
date=date).loc[lambda df: ~df['status'].isin(['Delisted', 'Unknown'])].
loc[:, 'order_book_id'].tolist())
