def _calc_synthetic_factor_loading(cls,
start_date,
end_date=None,
month_end=True,
save=False,
**kwargs):
"""
计算指定日期的样本个股的合成因子的载荷，并保存至因子数据库
Parameters
--------
:param start_date: datetime-like, str
开始日期
:param end_date: datetime-like, str，默认None
结束日期，如果为None，则只计算start_date日期的因子载荷
:param month_end: bool，默认True
只计算月末时点的因子载荷，该参数只在end_date不为None时有效，并且不论end_date是否为None，都会计算第一天的因子载荷
:param save: 是否保存至因子数据库，默认为False
:param kwargs:
'multi_proc': bool, True=采用多进程, False=采用单进程, 默认为False
'com_factors': list, 成分因子的类实例list
:return: 因子载荷，DataFrame
--------
因子载荷,DataFrame
0: ID, 证券ID，为索引
1: factorvalue, 因子载荷
"""
# 取得交易日序列
start_date = Utils.to_date(start_date)
if end_date is not None:
end_date = Utils.to_date(end_date)
trading_days_series = Utils.get_trading_days(start=start_date,
end=end_date)
else:
trading_days_series = Utils.get_trading_days(end=start_date,
ndays=1)
# 遍历交易日序列, 计算合成因子下各个成分因子的因子载荷
if 'multi_proc' not in kwargs:
kwargs['multi_proc'] = False
for calc_date in trading_days_series:
if month_end and (not Utils.is_month_end(calc_date)):
continue
# 计算各成分因子的因子载荷
# for com_factor in eval('risk_ct.' + cls.__name__.upper() + '_CT')['component']:
# factor = eval(com_factor + '()')
# factor.calc_factor_loading(start_date=calc_date, end_date=None, month_end=month_end, save=save, multi_proc=kwargs['multi_proc'])
for com_factor in kwargs['com_factors']:
com_factor.calc_factor_loading(start_date=calc_date,
end_date=None,
month_end=month_end,
save=save,
multi_proc=kwargs['multi_proc'])
# 计算合成因子
synthetic_factor = pd.DataFrame()
df_industry_classify = Utils.get_industry_classify() # 个股行业分类数据
for com_factor in eval('risk_ct.' + cls.__name__.upper() +
'_CT')['component']:
factor_path = os.path.join(
factor_ct.FACTOR_DB.db_path,
eval('risk_ct.' + com_factor + '_CT')['db_file'])
factor_loading = Utils.read_factor_loading(
factor_path,
Utils.datetimelike_to_str(calc_date, dash=False))
factor_loading.drop(columns='date', inplace=True)
factor_loading.rename(columns={'factorvalue': com_factor},
inplace=True)
# 添加行业分类数据
factor_loading = pd.merge(
left=factor_loading,
right=df_industry_classify[['id', 'ind_code']],
how='inner',
on='id')
# 取得含缺失值的因子载荷数据
missingdata_factor = factor_loading[
factor_loading[com_factor].isna()]
# 删除factor_loading中的缺失值
factor_loading.dropna(axis='index', how='any', inplace=True)
# 对factor_loading去极值、标准化
factor_loading = Utils.normalize_data(factor_loading,
id='id',
columns=com_factor,
treat_outlier=True,
weight='cap',
calc_date=calc_date)
# 把missingdata_factor中的缺失值替换为行业均值
ind_codes = set(missingdata_factor['ind_code'])
ind_mean_factor = {}
for ind_code in ind_codes:
ind_mean_factor[ind_code] = factor_loading[
factor_loading['ind_code'] ==
ind_code][com_factor].mean()
for idx, missingdata in missingdata_factor.iterrows():
missingdata_factor.loc[idx, com_factor] = ind_mean_factor[
missingdata['ind_code']]
# 把missingdata_factor和factor_loading合并
factor_loading = pd.concat(
[factor_loading, missingdata_factor])
# 删除ind_code列
factor_loading.drop(columns='ind_code', inplace=True)
# merge成分因子
if synthetic_factor.empty:
synthetic_factor = factor_loading
else:
synthetic_factor = pd.merge(left=synthetic_factor,
right=factor_loading,
how='inner',
on='id')
# 合成因子
synthetic_factor.set_index('id', inplace=True)
weight = pd.Series(
eval('risk_ct.' + cls.__name__.upper() + '_CT')['weight'])
synthetic_factor = (synthetic_factor * weight).sum(axis=1)
synthetic_factor.name = 'factorvalue'
synthetic_factor.index.name = 'id'
synthetic_factor = pd.DataFrame(synthetic_factor)
synthetic_factor.reset_index(inplace=True)
synthetic_factor['date'] = Utils.get_trading_days(start=calc_date,
ndays=2)[1]
# 保存synthetic_factor因子载荷
if save:
Utils.factor_loading_persistent(
cls._db_file,
Utils.datetimelike_to_str(calc_date, dash=False),
synthetic_factor.to_dict('list'),
['date', 'id', 'factorvalue'])
def calc_factor_loading_(cls,
start_date,
end_date=None,
month_end=True,
save=False,
**kwargs):
"""
计算指定日期的样本个股的因子载荷, 并保存至因子数据库
Parameters:
--------
:param start_date: datetime-like, str
开始日期, 格式: YYYY-MM-DD or YYYYMMDD
:param end_date: datetime-like, str
结束日期, 如果为None, 则只计算start_date日期的因子载荷, 格式: YYYY-MM-DD or YYYYMMDD
:param month_end: bool, 默认为True
如果为True, 则只计算月末时点的因子载荷
:param save: bool, 默认为True
是否保存至因子数据库
:param kwargs:
'multi_proc': bool, True=采用多进程, False=采用单进程, 默认为False
:return: dict
因子载荷数据
"""
# 取得交易日序列
start_date = Utils.to_date(start_date)
if end_date is not None:
end_date = Utils.to_date(end_date)
trading_days_series = Utils.get_trading_days(start=start_date,
end=end_date)
else:
trading_days_series = Utils.get_trading_days(end=start_date,
ndays=1)
# 遍历交易日序列, 计算growth因子下各个成分因子的因子载荷
if 'multi_proc' not in kwargs:
kwargs['multi_proc'] = False
for calc_date in trading_days_series:
if month_end and (not Utils.is_month_end(calc_date)):
continue
# 计算各成分因子的因子载荷
for com_factor in risk_ct.GROWTH_CT.component:
factor = eval(com_factor + '()')
factor.calc_factor_loading(start_date=calc_date,
end_date=None,
month_end=month_end,
save=save,
multi_proc=kwargs['multi_proc'])
# 合成Growth因子载荷
growth_factor = pd.DataFrame()
df_industry_classify = Utils.get_industry_classify() # 个股行业分类数据
for com_factor in risk_ct.GROWTH_CT.component:
factor_path = os.path.join(
factor_ct.FACTOR_DB.db_path,
eval('risk_ct.' + com_factor + '_CT')['db_file'])
factor_loading = Utils.read_factor_loading(
factor_path,
Utils.datetimelike_to_str(calc_date, dash=False))
factor_loading.drop(columns='date', inplace=True)
# factor_loading[com_factor] = Utils.normalize_data(Utils.clean_extreme_value(np.array(factor_loading['factorvalue']).reshape((len(factor_loading), 1))))
# factor_loading.drop(columns='factorvalue', inplace=True)
factor_loading.rename(columns={'factorvalue': com_factor},
inplace=True)
# 添加行业分类数据
factor_loading = pd.merge(
left=factor_loading,
right=df_industry_classify[['id', 'ind_code']],
how='inner',
on='id')
# 取得含缺失值的因子载荷数据
missingdata_factor = factor_loading[
factor_loading[com_factor].isna()]
# 删除factor_loading中的缺失值
factor_loading.dropna(axis='index', how='any', inplace=True)
# 对factor_loading去极值、标准化
factor_loading = Utils.normalize_data(factor_loading,
id='id',
columns=com_factor,
treat_outlier=True,
weight='cap',
calc_date=calc_date)
# 把missingdata_factor中的缺失值替换为行业均值
ind_codes = set(missingdata_factor['ind_code'])
ind_mean_factor = {}
for ind_code in ind_codes:
ind_mean_factor[ind_code] = factor_loading[
factor_loading['ind_code'] ==
ind_code][com_factor].mean()
for idx, missingdata in missingdata_factor.iterrows():
missingdata_factor.loc[idx, com_factor] = ind_mean_factor[
missingdata['ind_code']]
# 把missingdata_factor和factor_loading合并
factor_loading = pd.concat(
[factor_loading, missingdata_factor])
# 删除ind_code列
factor_loading.drop(columns='ind_code', inplace=True)
if growth_factor.empty:
growth_factor = factor_loading
else:
growth_factor = pd.merge(left=growth_factor,
right=factor_loading,
how='inner',
on='id')
# # 读取个股行业分类数据, 添加至growth_factor中
# df_industry_classify = Utils.get_industry_classify()
# growth_factor = pd.merge(left=growth_factor, right=df_industry_classify[['id', 'ind_code']])
# # 取得含缺失值的因子载荷数据
# missingdata_factor = growth_factor.loc[[ind for ind, data in growth_factor.iterrows() if data.hasnans]]
# # 删除growth_factot中的缺失值
# growth_factor.dropna(axis='index', how='any', inplace=True)
# # 对growth_factor去极值、标准化
# growth_factor = Utils.normalize_data(growth_factor, id='id', columns=risk_ct.GROWTH_CT.component, treat_outlier=True, weight='cap', calc_date=calc_date)
# # 把missingdata_factor中的缺失值替换为行业均值
# ind_codes = set(missingdata_factor['ind_code'])
# ind_mean_factor = {}
# for ind_code in ind_codes:
# ind_mean_factor[ind_code] = growth_factor[growth_factor['ind_code'] == ind_code].mean()
# missingdata_label = {ind: missingdata_factor.columns[missingdata.isna()].tolist() for ind, missingdata in missingdata_factor.iterrows()}
# for ind, cols in missingdata_label.items():
# missingdata_factor.loc[ind, cols] = ind_mean_factor[missingdata_factor.loc[ind, 'ind_code']][cols]
# # 把missingdata_factor和growth_factor合并
# growth_factor = pd.concat([growth_factor, missingdata_factor])
# # 删除ind_code列
# growth_factor.drop(columns='ind_code', inplace=True)
# 合成Growth因子
growth_factor.set_index('id', inplace=True)
weight = pd.Series(risk_ct.GROWTH_CT.weight)
growth_factor = (growth_factor * weight).sum(axis=1)
growth_factor.name = 'factorvalue'
growth_factor.index.name = 'id'
growth_factor = pd.DataFrame(growth_factor)
growth_factor.reset_index(inplace=True)
growth_factor['date'] = Utils.get_trading_days(start=calc_date,
ndays=2)[1]
# 保存growth因子载荷
if save:
Utils.factor_loading_persistent(
cls._db_file,
Utils.datetimelike_to_str(calc_date, dash=False),
growth_factor.to_dict('list'),
['date', 'id', 'factorvalue'])
def get_dependent_factors(cls, date):
"""
计算用于因子提纯的相关性因子值，包换行业、规模、价值、成长、短期动量、长期动量
Parameters:
--------
:param date: datetime-like or str
日期
:return: pd.DataFrame
index为个股代码, columns=[28个申万一级行业,规模(scale),价值(value),成长(growth),短期动量(short_momentum),长期动量(long_momentum)]
"""
str_date = Utils.to_date(date).strftime('%Y%m%d')
# 1. 行业因子
# 1.1. 读取行业分类信息
df_industry_calssify = Utils.get_industry_classify()
df_industry_calssify = df_industry_calssify.set_index('id')
# 1.2. 构建行业分裂哑变量
df_industry_dummies = pd.get_dummies(df_industry_calssify['ind_code'])
# 2. 规模因子
# 2.1. 读取规模因子
scale_factor_path = os.path.join(factor_ct.FACTOR_DB.db_path,
factor_ct.SCALE_CT.db_file)
df_scale_raw = Utils.read_factor_loading(scale_factor_path,
str_date,
nan_value=0)
# 2.2. 规模因子去极值、标准化
scale_cleaned_arr = Utils.clean_extreme_value(
np.array(df_scale_raw[['LnLiquidMktCap', 'LnTotalMktCap']]))
scale_normalized_arr = Utils.normalize_data(scale_cleaned_arr)
# 2.3. 规模因子降维
scale_factor_arr = np.mean(scale_normalized_arr, axis=1)
scale_factor = Series(scale_factor_arr, index=df_scale_raw['id'])
# 3. 价值因子
# 3.1. 读取价值因子
value_factor_path = os.path.join(factor_ct.FACTOR_DB.db_path,
factor_ct.VALUE_CT.db_file)
df_value_raw = Utils.read_factor_loading(value_factor_path,
str_date,
nan_value=0)
# 3.2. 价值因子去极值、标准化
value_cleaned_arr = Utils.clean_extreme_value(
np.array(df_value_raw[['ep_ttm', 'bp_lr', 'ocf_ttm']]))
value_normalized_arr = Utils.normalize_data(value_cleaned_arr)
# 3.3. 价值因子降维
value_factor_arr = np.mean(value_normalized_arr, axis=1)
value_factor = Series(value_factor_arr, index=df_value_raw['id'])
# 4. 成长因子
# 4.1. 读取成长因子
growth_factor_path = os.path.join(factor_ct.FACTOR_DB.db_path,
factor_ct.GROWTH_CT.db_file)
df_growth_raw = Utils.read_factor_loading(growth_factor_path,
str_date,
nan_value=0)
# 4.2. 成长因子去极值、标准化
growth_cleaned_arr = Utils.clean_extreme_value(
np.array(df_growth_raw[['npg_ttm', 'opg_ttm']]))
growth_normalized_arr = Utils.normalize_data(growth_cleaned_arr)
# 4.3. 成长因子降维
growth_factor_arr = np.mean(growth_normalized_arr, axis=1)
growth_factor = Series(growth_factor_arr, index=df_growth_raw['id'])
# 5. 动量因子
# 5.1. 读取动量因子
mom_factor_path = os.path.join(factor_ct.FACTOR_DB.db_path,
factor_ct.MOMENTUM_CT.db_file)
df_mom_raw = Utils.read_factor_loading(mom_factor_path,
str_date,
nan_value=0)
# 5.2. 动量因子去极值、标准化
short_term_mom_header = [
'short_term_' + d
for d in factor_ct.MOMENTUM_CT.short_term_days.split('|')
]
short_mom_cleaned_arr = Utils.clean_extreme_value(
np.array(df_mom_raw[short_term_mom_header]))
short_mom_normalized_arr = Utils.normalize_data(short_mom_cleaned_arr)
long_term_mom_header = [
'long_term_' + d
for d in factor_ct.MOMENTUM_CT.long_term_days.split('|')
]
long_mom_cleaned_arr = Utils.clean_extreme_value(
np.array(df_mom_raw[long_term_mom_header]))
long_mom_normalized_arr = Utils.normalize_data(long_mom_cleaned_arr)
# 5.3. 动量因子降维
short_mom_arr = np.mean(short_mom_normalized_arr, axis=1)
short_mom = Series(short_mom_arr, index=df_mom_raw['id'])
long_mom_arr = np.mean(long_mom_normalized_arr, axis=1)
long_mom = Series(long_mom_arr, index=df_mom_raw['id'])
# 拼接除行业因子外的因子
df_style_factor = pd.concat(
[scale_factor, value_factor, growth_factor, short_mom, long_mom],
axis=1,
keys=['scale', 'value', 'growth', 'short_mom', 'long_mom'],
join='inner')
# 再拼接行业因子
df_dependent_factor = pd.concat([df_industry_dummies, df_style_factor],
axis=1,
join='inner')
return df_dependent_factor
