def calc_factor_loading_(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷, 并保存至因子数据库 Parameters: -------- :param start_date: datetime-like, str 开始日期, 格式: YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期, 如果为None, 则只计算start_date日期的因子载荷, 格式: YYYY-MM-DD or YYYYMMDD :param month_end: bool, 默认为True 如果为True, 则只计算月末时点的因子载荷 :param save: bool, 默认为True 是否保存至因子数据库 :param kwargs: 'multi_proc': bool, True=采用多进程, False=采用单进程, 默认为False :return: dict 因子载荷数据 """ # 取得交易日序列 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) # 遍历交易日序列, 计算ResVolatility因子下各个成分因子的因子载荷 if 'multi_proc' not in kwargs: kwargs['multi_proc'] = False for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue # 计算各成分因子的因子载荷 for com_factor in risk_ct.RESVOLATILITY_CT.component: factor = eval(com_factor + '()') factor.calc_factor_loading(start_date=calc_date, end_date=None, month_end=month_end, save=save, multi_proc=kwargs['multi_proc']) # 合成ResVolatility因子载荷 resvol_factor = pd.DataFrame() for com_factor in risk_ct.RESVOLATILITY_CT.component: factor_path = os.path.join(factor_ct.FACTOR_DB.db_path, eval('risk_ct.' + com_factor + '_CT')['db_file']) factor_loading = Utils.read_factor_loading(factor_path, Utils.datetimelike_to_str(calc_date, dash=False)) factor_loading.drop(columns='date', inplace=True) factor_loading[com_factor] = Utils.normalize_data(Utils.clean_extreme_value(np.array(factor_loading['factorvalue']).reshape((len(factor_loading), 1)))) factor_loading.drop(columns='factorvalue', inplace=True) if resvol_factor.empty: resvol_factor = factor_loading else: resvol_factor = pd.merge(left=resvol_factor, right=factor_loading, how='inner', on='id') resvol_factor.set_index('id', inplace=True) weight = pd.Series(risk_ct.RESVOLATILITY_CT.weight) resvol_factor = (resvol_factor * weight).sum(axis=1) resvol_factor.name = 'factorvalue' resvol_factor.index.name = 'id' resvol_factor = pd.DataFrame(resvol_factor) resvol_factor.reset_index(inplace=True) resvol_factor['date'] = Utils.get_trading_days(start=calc_date, ndays=2)[1] # 保存ResVolatility因子载荷 if save: Utils.factor_loading_persistent(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), resvol_factor.to_dict('list'),['date', 'id', 'factorvalue'])
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷, 并保存至因子数据库 Parameters: -------- :param start_date: datetime-like, str 开始日期, 格式:YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期, 如果为None, 则只计算start_date日期的因子载荷, 格式: YYYY-MM-DD or YYYYMMDD :param month_end: bool, 默认为True 如果为True, 则只计算月末时点的因子载荷 :param save: bool, 默认为True 是否保存至因子数据库 :param kwargs: :return: dict 因子载荷数据 """ # 取得交易日序列 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) # 遍历交易日序列, 计算NLSIZE因子载荷 dict_nlsize = None for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue logging.info('[%s] Calc NLSIZE factor loading.' % Utils.datetimelike_to_str(calc_date)) # 读取Size因子载荷数据 lncap_data_path = os.path.join(factor_ct.FACTOR_DB.db_path, '{}_{}.csv'.format(risk_ct.SIZE_CT.db_file, Utils.datetimelike_to_str(calc_date, dash=False))) if not os.path.exists(lncap_data_path): logging.info('[%s] 的Size因子载荷数据不存在.' % Utils.datetimelike_to_str(calc_date)) continue df_lncap = pd.read_csv(lncap_data_path, header=0) # Size因子数组 arr_size = np.array(df_lncap['factorvalue']) # Size因子三次方数组 arr_size_cube = arr_size ** 3 # 相对Size因子正交化 model = sm.OLS(arr_size_cube, arr_size) result = model.fit() # 对残差值进行缩尾处理和标准化 n = len(result.resid) arr_resid = result.resid # arr_resid = result.resid.reshape(n, 1) # arr_resid_winsorized = Utils.clean_extreme_value(arr_resid) # arr_resid_standardized = Utils.normalize_data(arr_resid_winsorized) # 保存NLSIZE因子载荷数据 dict_nlsize = dict({'date': df_lncap['date'].values, 'id': df_lncap['id'].values, 'factorvalue': arr_resid}) if save: Utils.factor_loading_persistent(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), dict_nlsize, ['date', 'id', 'factorvalue']) return dict_nlsize
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本股的因子载荷,并保存至因子数据库 Parameters: -------- :param start_date: datetime-like or str 开始日期,格式:YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like or str 结束日期,格式:YYYY-MM-DD or YYYYMMDD :param month_end: bool, 默认True 如果为True,则只计算月末时点的因子载荷;否则每个交易日都计算 :param save: bool, 默认False 是否保存至因子数据库 :return: 因子载荷,DataFrame -------- 因子载荷,DataFrame 0. date: 日期 1. id: 证券symbol 2. LnTotalMktCap: 总市值对数值 3. LnLiquidMktCap: 流通市值对数值 """ # 取得交易日序列股票基本信息表 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列,计算规模因子值 dict_scale = None for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue dict_scale = { 'date': [], 'id': [], 'LnTotalMktCap': [], 'LnLiquidMktCap': [] } # 遍历个股,计算个股规模因子值 s = (calc_date - datetime.timedelta(days=90)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] # 采用单进程进行计算规模因子 # for _, stock_info in stock_basics.iterrows(): # scale_data = cls._calc_factor_loading(stock_info.symbol, calc_date) # if scale_data is not None: # logging.info("[%s] %s's total mkt cap = %.0f, liquid mkt cap = %.0f" % (calc_date.strftime('%Y-%m-%d'), stock_info.symbol, scale_data.LnTotalMktCap, scale_data.LnLiquidMktCap)) # dict_scale['id'].append(Utils.code_to_symbol(stock_info.symbol)) # dict_scale['LnTotalMktCap'].append(round(scale_data.LnTotalMktCap, 4)) # dict_scale['LnLiquidMktCap'].append(round(scale_data.LnLiquidMktCap, 4)) # 采用多进程并行计算规模因子 q = Manager().Queue() # 队列,用于进程间通信,存储每个进程计算的规模因子值 p = Pool(4) # 进程池,最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=( stock_info.symbol, calc_date, q, )) p.close() p.join() while not q.empty(): scale_data = q.get(True) dict_scale['id'].append(scale_data[0]) dict_scale['LnTotalMktCap'].append(round(scale_data[1], 4)) dict_scale['LnLiquidMktCap'].append(round(scale_data[2], 4)) date_label = Utils.get_trading_days(start=calc_date, ndays=2)[1] dict_scale['date'] = [date_label] * len(dict_scale['id']) # 保存规模因子载荷至因子数据库 if save: Utils.factor_loading_persistent(cls._db_file, calc_date.strftime('%Y%m%d'), dict_scale) # 休息60秒 logging.info('Suspending for 60s.') time.sleep(60) return dict_scale
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷,并保存至因子数据库 Parameters -------- :param start_date: datetime-like, str 开始日期 :param end_date: datetime-like, str,默认None 结束日期,如果为None,则只计算start_date日期的因子载荷 :param month_end: bool,默认True 只计算月末时点的因子载荷 :param save: 是否保存至因子数据库,默认为False :return: 因子载荷,DataFrame -------- 因子载荷,DataFrame 0: ID, 证券ID,为索引 1: factorvalue, 因子载荷 如果end_date=None,返回start_date对应的因子载荷数据 如果end_date!=None,返回最后一天的对应的因子载荷数据 如果没有计算数据,返回None """ # 0.取得交易日序列 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) # 取得样本个股信息 all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列,计算SMartQ因子载荷 dict_factor = None for calc_date in trading_days_series: dict_factor = {'id': [], 'factorvalue': []} if month_end and (not Utils.is_month_end(calc_date)): continue # 1.获取用于读取分钟行情的交易日列表(过去30天的交易日列表,降序排列) # trading_days = _get_trading_days(calc_date, 30) # trading_days = Utils.get_trading_days(end=calc_date, ndays=30, ascending=False) # 2.取得样本个股信息 # stock_basics = ts.get_stock_basics() s = (calc_date - datetime.timedelta(days=90)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] # 3.遍历样本个股代码,计算Smart_Q因子载荷值 dict_factor = {'id': [], 'factorvalue': []} # 采用单进程进行计算 # for _, stock_info in stock_basics.iterrows(): # # code = '%s%s' % ('SH' if code[:2] == '60' else 'SZ', code) # factor_loading = cls._calc_factor_loading(stock_info.symbol, calc_date) # print("[%s]Calculating %s's SmartMoney factor loading = %.4f." % (calc_date.strftime('%Y-%m-%d'), stock_info.symbol, -1.0 if factor_loading is None else factor_loading)) # if factor_loading is not None: # # df_factor.ix[code, 'factorvalue'] = factor_loading # dict_factor['id'].append(Utils.code_to_symbol(stock_info.symbol)) # dict_factor['factorvalue'].append(factor_loading) # 采用多进程并行计算SmartQ因子载荷 q = Manager().Queue() # 队列,用于进程间通信,存储每个进程计算的因子载荷值 p = Pool(4) # 进程池,最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=( stock_info.symbol, calc_date, q, )) p.close() p.join() while not q.empty(): smart_q = q.get(True) dict_factor['id'].append(smart_q[0]) dict_factor['factorvalue'].append(smart_q[1]) date_label = Utils.get_trading_days(calc_date, ndays=2)[1] dict_factor['date'] = [date_label] * len(dict_factor['id']) # 4.保存因子载荷至因子数据库 if save: # db = shelve.open(cls._db_file, flag='c', protocol=None, writeback=False) # try: # db[calc_date.strftime('%Y%m%d')] = df_factor # finally: # db.close() Utils.factor_loading_persistent(cls._db_file, calc_date.strftime('%Y%m%d'), dict_factor) # 休息300秒 logging.info('Suspending for 360s.') time.sleep(360) return dict_factor
def calc_factor_loading_(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷, 并保存至因子数据库 Parameters: -------- :param start_date: datetime-like, str 开始日期, 格式: YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期, 如果为None, 则只计算start_date日期的因子载荷, 格式: YYYY-MM-DD or YYYYMMDD :param month_end: bool, 默认为True 如果为True, 则只计算月末时点的因子载荷 :param save: bool, 默认为True 是否保存至因子数据库 :param kwargs: 'multi_proc': bool, True=采用多进程, False=采用单进程, 默认为False :return: dict 因子载荷数据 """ # 取得交易日序列 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) # 遍历交易日序列, 计算growth因子下各个成分因子的因子载荷 if 'multi_proc' not in kwargs: kwargs['multi_proc'] = False for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue # 计算各成分因子的因子载荷 for com_factor in risk_ct.GROWTH_CT.component: factor = eval(com_factor + '()') factor.calc_factor_loading(start_date=calc_date, end_date=None, month_end=month_end, save=save, multi_proc=kwargs['multi_proc']) # 合成Growth因子载荷 growth_factor = pd.DataFrame() df_industry_classify = Utils.get_industry_classify() # 个股行业分类数据 for com_factor in risk_ct.GROWTH_CT.component: factor_path = os.path.join( factor_ct.FACTOR_DB.db_path, eval('risk_ct.' + com_factor + '_CT')['db_file']) factor_loading = Utils.read_factor_loading( factor_path, Utils.datetimelike_to_str(calc_date, dash=False)) factor_loading.drop(columns='date', inplace=True) # factor_loading[com_factor] = Utils.normalize_data(Utils.clean_extreme_value(np.array(factor_loading['factorvalue']).reshape((len(factor_loading), 1)))) # factor_loading.drop(columns='factorvalue', inplace=True) factor_loading.rename(columns={'factorvalue': com_factor}, inplace=True) # 添加行业分类数据 factor_loading = pd.merge( left=factor_loading, right=df_industry_classify[['id', 'ind_code']], how='inner', on='id') # 取得含缺失值的因子载荷数据 missingdata_factor = factor_loading[ factor_loading[com_factor].isna()] # 删除factor_loading中的缺失值 factor_loading.dropna(axis='index', how='any', inplace=True) # 对factor_loading去极值、标准化 factor_loading = Utils.normalize_data(factor_loading, id='id', columns=com_factor, treat_outlier=True, weight='cap', calc_date=calc_date) # 把missingdata_factor中的缺失值替换为行业均值 ind_codes = set(missingdata_factor['ind_code']) ind_mean_factor = {} for ind_code in ind_codes: ind_mean_factor[ind_code] = factor_loading[ factor_loading['ind_code'] == ind_code][com_factor].mean() for idx, missingdata in missingdata_factor.iterrows(): missingdata_factor.loc[idx, com_factor] = ind_mean_factor[ missingdata['ind_code']] # 把missingdata_factor和factor_loading合并 factor_loading = pd.concat( [factor_loading, missingdata_factor]) # 删除ind_code列 factor_loading.drop(columns='ind_code', inplace=True) if growth_factor.empty: growth_factor = factor_loading else: growth_factor = pd.merge(left=growth_factor, right=factor_loading, how='inner', on='id') # # 读取个股行业分类数据, 添加至growth_factor中 # df_industry_classify = Utils.get_industry_classify() # growth_factor = pd.merge(left=growth_factor, right=df_industry_classify[['id', 'ind_code']]) # # 取得含缺失值的因子载荷数据 # missingdata_factor = growth_factor.loc[[ind for ind, data in growth_factor.iterrows() if data.hasnans]] # # 删除growth_factot中的缺失值 # growth_factor.dropna(axis='index', how='any', inplace=True) # # 对growth_factor去极值、标准化 # growth_factor = Utils.normalize_data(growth_factor, id='id', columns=risk_ct.GROWTH_CT.component, treat_outlier=True, weight='cap', calc_date=calc_date) # # 把missingdata_factor中的缺失值替换为行业均值 # ind_codes = set(missingdata_factor['ind_code']) # ind_mean_factor = {} # for ind_code in ind_codes: # ind_mean_factor[ind_code] = growth_factor[growth_factor['ind_code'] == ind_code].mean() # missingdata_label = {ind: missingdata_factor.columns[missingdata.isna()].tolist() for ind, missingdata in missingdata_factor.iterrows()} # for ind, cols in missingdata_label.items(): # missingdata_factor.loc[ind, cols] = ind_mean_factor[missingdata_factor.loc[ind, 'ind_code']][cols] # # 把missingdata_factor和growth_factor合并 # growth_factor = pd.concat([growth_factor, missingdata_factor]) # # 删除ind_code列 # growth_factor.drop(columns='ind_code', inplace=True) # 合成Growth因子 growth_factor.set_index('id', inplace=True) weight = pd.Series(risk_ct.GROWTH_CT.weight) growth_factor = (growth_factor * weight).sum(axis=1) growth_factor.name = 'factorvalue' growth_factor.index.name = 'id' growth_factor = pd.DataFrame(growth_factor) growth_factor.reset_index(inplace=True) growth_factor['date'] = Utils.get_trading_days(start=calc_date, ndays=2)[1] # 保存growth因子载荷 if save: Utils.factor_loading_persistent( cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), growth_factor.to_dict('list'), ['date', 'id', 'factorvalue'])
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷,并保存至因子数据库 Parameters: -------- :param start_date: datetime-like, str 开始日期,格式:YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期,格式:YYYY-MM-DD or YYYYMMDD 如果为None,则只计算start_date日期的因子载荷 :param month_end:bool, 默认True 如果为True,则只结算月末时点的因子载荷 :param save: bool, 默认False 是否保存至因子数据库 :return: 因子载荷,DataFrame -------- 因子载荷,DataFrame 0. date: 日期 1. id: 证券symbol 2. short_term_0: 第一个短期动量因子 3. short_term_1: 第二个短期动量因子 4. long_term_0: 第一个长期动量因子 5. long_term_1: 第二个长期动量因子 """ # 取得交易日序列及股票基本信息表 # start_date = Utils.to_date(start_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列,计算动量因子 dict_momentum = None momentum_label = cls.momentum_label() for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue dict_momentum = {'date': [], 'id': []} for label in momentum_label: dict_momentum[label] = [] # 遍历个股,计算个股动量因子 s = (calc_date - datetime.timedelta(days=90)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] # 采用单进程进行计算 # for _, stock_info in stock_basics.iterrows(): # momentum_data = cls._calc_factor_loading(stock_info.symbol, calc_date) # if momentum_data is not None: # logging.info("[%s] calc %s's momentum factor loading." % (calc_date.strftime('%Y-%m-%d'), stock_info.symbol)) # dict_momentum['id'].append(Utils.code_to_symbol(stock_info.symbol)) # for label in momentum_label: # dict_momentum[label].append(momentum_data[label]) # 采用多进程并行计算动量因子载荷 q = Manager().Queue() # 队列,用于进程间通信,存储每个进程计算的因子载荷 p = Pool(4) # 进程池,最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=( stock_info.symbol, calc_date, q, )) p.close() p.join() while not q.empty(): momentum_data = q.get(True) dict_momentum['id'].append(momentum_data['id']) for label in momentum_label: dict_momentum[label].append(momentum_data[label]) date_label = Utils.get_trading_days(start=calc_date, ndays=2)[1] dict_momentum['date'] = [date_label] * len(dict_momentum['id']) # 保存因子载荷至因子数据库 if save: Utils.factor_loading_persistent(cls._db_file, calc_date.strftime('%Y%m%d'), dict_momentum) # 休息60秒 logging.info('Suspending for 60s.') time.sleep(60) return dict_momentum
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷,并保存至因子数据库 Parameters: -------- :param start_date: datetime-like, str 开始日期,格式:YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str; 默认为None 结束日期,格式:YYYY-MM-DD or YYYYMMDD 如果为None,则只计算start_date日期的因子载荷 :param month_end: bool, 默认True 如果为True,则只计算月末时点的因子载荷 :param save: bool, 默认False 是否保存至因子数据库 :return: 因子载荷,pd.DataFrame -------- 因子载荷,pd.DataFrame 0. date: 日期 1. id: 日期 2. npg_ttm: 净利润增长率_TTM 3. opg_ttm: 营业收入增长率_TTM """ # 取得交易日序列及股票基本信息表 trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列,计算价值因子载荷 dict_growth = None for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue dict_growth = {'date': [], 'id': [], 'npg_ttm': [], 'opg_ttm': []} # 遍历个股,计算个股成长因子载荷 s = (calc_date - datetime.timedelta(days=90)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] # 采用单进程进行计算成长因子 # for _, stock_info in stock_basics.iterrows(): # logging.info("[%s] calc %s's growth factor loading." % (calc_date.strftime('%Y-%m-%d'), stock_info.symbol)) # growth_data = cls._calc_factor_loading(stock_info.symbol, calc_date) # if growth_data is not None: # dict_growth['id'].append(Utils.code_to_symbol(stock_info.symbol)) # dict_growth['npg_ttm'].append(growth_data['npg_ttm']) # dict_growth['opg_ttm'].append(growth_data['opg_ttm']) # 采用多进程并行计算成长因子 q = Manager().Queue() # 队列,用于进程间通信,存储每个进程计算的因子载荷 p = Pool(4) # 进程池,最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=(stock_info.symbol, calc_date, q,)) p.close() p.join() while not q.empty(): growth_data = q.get(True) dict_growth['id'].append(growth_data['id']) dict_growth['npg_ttm'].append(growth_data['npg_ttm']) dict_growth['opg_ttm'].append(growth_data['opg_ttm']) date_label = Utils.get_trading_days(start=calc_date, ndays=2)[1] dict_growth['date'] = [date_label] * len(dict_growth['id']) # 保存因子载荷至因子数据库 if save: columns = ['date', 'id', 'npg_ttm', 'opg_ttm'] Utils.factor_loading_persistent(cls._db_file, calc_date.strftime('%Y%m%d'), dict_growth, columns) # 休息120秒 logging.info('Suspending for 120s.') time.sleep(120) return dict_growth
def _calc_Orthogonalized_factorloading(factor_name, start_date, end_date=None, month_end=True, save=False): """ 计算alpha因子经正交化后的因子载荷 Parameters: -------- :param factor_name: str alpha因子名称, e.g: SmartMoney :param start_date: datetime-like, str 开始日期, e.g: YYYY-MM-DD, YYYYMMDD :param end_date: datetime-like, str, 默认None 结束日期, e.g: YYYY-MM-DD, YYYYMMDD :param month_end: bool, 默认True 是否只计算月末日期的因子载荷 :param save: bool, 默认False 是否保存计算结果 :return: dict -------- 因子经正交化后的因子载荷 0. date, 为计算日期的下一个交易日 1. id, 证券代码 2. factorvalue, 因子载荷 如果end_date=None,返回start_date对应的因子载荷数据 如果end_date!=None,返回最后一天的对应的因子载荷数据 如果没有计算数据,返回None """ start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) CRiskModel = Barra() orthog_factorloading = {} for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue # 读取目标因子原始载荷经标准化后的载荷值 target_factor_path = os.path.join( SETTINGS.FACTOR_DB_PATH, eval('alphafactor_ct.' + factor_name.upper() + '_CT')['db_file'], 'standardized', factor_name) df_targetfactor_loading = Utils.read_factor_loading( target_factor_path, Utils.datetimelike_to_str(calc_date, dash=False), drop_na=True) df_targetfactor_loading.drop(columns='date', inplace=True) df_targetfactor_loading.rename(columns={'factorvalue': factor_name}, inplace=True) # 读取风险模型中的风格因子载荷矩阵 df_stylefactor_loading = CRiskModel.get_StyleFactorloading_matrix( calc_date) df_stylefactor_loading.renmae(columns={'code': 'id'}, inplace=True) # 读取alpha因子载荷矩阵数据(经正交化后的载荷值) df_alphafactor_loading = pd.DataFrame() for alphafactor_name in alphafactor_ct.ALPHA_FACTORS: if alphafactor_name == factor_name: break factorloading_path = os.path.join( SETTINGS.FACTOR_DB_PATH, eval('alphafactor_ct.' + alphafactor_name.upper() + '_CT')['db_file'], 'orthogonalized', alphafactor_name) factor_loading = Utils.read_factor_loading( factorloading_path, Utils.datetimelike_to_str(calc_date, dash=False), drop_na=True) factor_loading.drop(columns='date', inplace=True) factor_loading.rename(columns={'factorvalue': alphafactor_name}, inplace=True) if df_alphafactor_loading.empty: df_alphafactor_loading = factor_loading else: df_alphafactor_loading = pd.merge(left=df_alphafactor_loading, right=factor_loading, how='inner', on='id') # 合并目标因子载荷、风格因子载荷与alpha因子载荷 df_factorloading = pd.merge(left=df_targetfactor_loading, right=df_stylefactor_loading, how='inner', on='id') if not df_alphafactor_loading.empty: df_factorloading = pd.merge(left=df_stylefactor_loading, right=df_alphafactor_loading, how='inner', on='id') # 构建目标因子载荷向量、风格与alpha因子载荷矩阵 df_factorloading.set_index('id', inplace=True) arr_targetfactor_loading = np.array(df_factorloading[factor_name]) stylealphafactor_names = df_factorloading.columns.tolist() stylealphafactor_names.remove(factor_name) arr_stylealphafactor_loading = np.array( df_factorloading[stylealphafactor_names]) # 将arr_targetfactor_loading对arr_stylealphafactor_loading进行截面回归, 得到的残差即为正交化后的因子载荷 Y = arr_targetfactor_loading X = sm.add_constant(arr_stylealphafactor_loading) model = sm.OLS(Y, X) results = model.fit() datelabel = Utils.get_trading_days(start=calc_date, ndays=2)[1] orthog_factorloading = { 'date': [datelabel] * len(results.resid), 'id': df_factorloading.index.tolist(), 'factorvalue': results.resid } # 保存正交化后的因子载荷 if save: orthog_factorloading_path = os.path.join( SETTINGS.FACTOR_DB_PATH, eval('alphafactor_ct.' + factor_name.upper() + '_CT')['db_file'], 'orthogonalized', factor_name) Utils.factor_loading_persistent( orthog_factorloading_path, Utils.datetimelike_to_str(calc_date, dash=False), orthog_factorloading, ['date', 'id', 'factorvalue']) return orthog_factorloading
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷, 并保存至因子数据库 Parameters: -------- :param start_date: datetime-like, str 开始日期, 格式: YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期, 如果为None, 则只计算start_date日期的因子载荷, 格式: YYYY-MM-DD or YYYYMMDD :param month_end: bool, 默认为True 如果为True, 则只计算月末时点的因子载荷 :param save: bool, 默认为True 是否保存至因子数据库 :param kwargs: 'multi_proc': bool, True=采用多进程, False=采用单进程, 默认为False :return: dict 因子载荷 """ # 取得交易日序列及股票基本信息表 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列, 计算DASTD因子载荷 dict_dastd = None for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue logging.info('[%s] Calc DASTD factor loading.' % Utils.datetimelike_to_str(calc_date)) # 遍历个股, 计算个股的DASTD因子值 s = (calc_date - datetime.timedelta(days=risk_ct.DASTD_CT.listed_days)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] ids = [] # 个股代码list dastds = [] # DASTD因子值list if 'multi_proc' not in kwargs: kwargs['multi_proc'] = False if not kwargs['multi_proc']: # 采用单进程计算DASTD因子值 for _, stock_info in stock_basics.iterrows(): logging.info("[%s] Calc %s's DASTD factor loading." % (calc_date.strftime('%Y-%m-%d'), stock_info.symbol)) dastd_data = cls._calc_factor_loading(stock_info.symbol, calc_date) if dastd_data is None: ids.append(Utils.code_to_symbol(stock_info.symbol)) dastds.append(np.nan) else: ids.append(dastd_data['code']) dastds.append(dastd_data['dastd']) else: # 采用多进程并行计算DASTD因子值 q = Manager().Queue() # 队列, 用于进程间通信, 存储每个进程计算的因子载荷 p = Pool(4) # 进程池, 最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=(stock_info.symbol, calc_date, q,)) p.close() p.join() while not q.empty(): dastd_data = q.get(True) ids.append(dastd_data['code']) dastds.append(dastd_data['dastd']) date_label = Utils.get_trading_days(start=calc_date, ndays=2)[1] dict_dastd = {'date': [date_label]*len(ids), 'id': ids, 'factorvalue': dastds} if save: Utils.factor_loading_persistent(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), dict_dastd, ['date', 'id', 'factorvalue']) # 暂停180秒 logging.info('Suspending for 180s.') # time.sleep(180) return dict_dastd
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷,并保存至因子数据库 Parameters -------- :param start_date: datetime-like, str 开始日期 :param end_date: datetime-like, str,默认None 结束日期,如果为None,则只计算start_date日期的因子载荷 :param month_end: bool,默认True 只计算月末时点的因子载荷,该参数只在end_date不为None时有效,并且不论end_date是否为None,都会计算第一天的因子载荷 :param save: 是否保存至因子数据库,默认为False :return: 因子载荷,DataFrame -------- 因子载荷,DataFrame 0: id, 证券ID 1: factorvalue, 因子载荷 如果end_date=None,返回start_date对应的因子载荷数据 如果end_date!=None,返回最后一天的对应的因子载荷数据 如果没有计算数据,返回None """ # 1.取得交易日序列及股票基本信息表 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 2.遍历交易日序列,计算APM因子载荷 dict_apm = None for calc_date in trading_days_series: dict_apm = {'date': [], 'id': [], 'factorvalue': []} if month_end and (not Utils.is_month_end(calc_date)): continue # 2.1.遍历个股,计算个股APM.stat统计量,过去20日收益率,分别放进stat_lst,ret20_lst列表中 s = (calc_date - datetime.timedelta(days=90)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] stat_lst = [] ret20_lst = [] symbol_lst = [] # 采用单进程计算 # for _, stock_info in stock_basics.iterrows(): # stat_i = cls._calc_factor_loading(stock_info.symbol, calc_date) # ret20_i = Utils.calc_interval_ret(stock_info.symbol, end=calc_date, ndays=20) # if stat_i is not None and ret20_i is not None: # stat_lst.append(stat_i) # ret20_lst.append(ret20_i) # symbol_lst.append(Utils.code_to_symbol(stock_info.symbol)) # logging.info('APM of %s = %f' % (stock_info.symbol, stat_i)) # 采用多进程并行计算 q = Manager().Queue() p = Pool(4) # 最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=( stock_info.symbol, calc_date, q, )) p.close() p.join() while not q.empty(): apm_value = q.get(True) symbol_lst.append(apm_value[0]) stat_lst.append(apm_value[1]) ret20_lst.append(apm_value[2]) assert len(stat_lst) == len(ret20_lst) assert len(stat_lst) == len(symbol_lst) # 2.2.构建APM因子 # 2.2.1.将统计量stat对动量因子ret20j进行截面回归:stat_j = \beta * Ret20_j + \epsilon_j # 残差向量即为对应个股的APM因子 # 截面回归之前,先对stat统计量和动量因子进行去极值和标准化处理 stat_arr = np.array(stat_lst).reshape((len(stat_lst), 1)) ret20_arr = np.array(ret20_lst).reshape((len(ret20_lst), 1)) stat_arr = Utils.clean_extreme_value(stat_arr) stat_arr = Utils.normalize_data(stat_arr) ret20_arr = Utils.clean_extreme_value(ret20_arr) ret20_arr = Utils.normalize_data(ret20_arr) # 回归分析 # ret20_arr = sm.add_constant(ret20_arr) apm_model = sm.OLS(stat_arr, ret20_arr) apm_result = apm_model.fit() apm_lst = list(np.around(apm_result.resid, 6)) # amp因子载荷精确到6位小数 assert len(apm_lst) == len(symbol_lst) # 2.2.2.构造APM因子字典,并持久化 date_label = Utils.get_trading_days(calc_date, ndays=2)[1] dict_apm = { 'date': [date_label] * len(symbol_lst), 'id': symbol_lst, 'factorvalue': apm_lst } if save: Utils.factor_loading_persistent(cls._db_file, calc_date.strftime('%Y%m%d'), dict_apm) # 2.3.构建PureAPM因子 # 将stat_arr转换为DataFrame, 此时的stat_arr已经经过了去极值和标准化处理 df_stat = DataFrame(stat_arr, index=symbol_lst, columns=['stat']) # 取得提纯的因变量因子 df_dependent_factor = cls.get_dependent_factors(calc_date) # 将df_stat和因变量因子拼接 df_data = pd.concat([df_stat, df_dependent_factor], axis=1, join='inner') # OLS回归,提纯APM因子 arr_data = np.array(df_data) pure_apm_model = sm.OLS(arr_data[:, 0], arr_data[:, 1:]) pure_apm_result = pure_apm_model.fit() pure_apm_lst = list(np.around(pure_apm_result.resid, 6)) pure_symbol_lst = list(df_data.index) assert len(pure_apm_lst) == len(pure_symbol_lst) # 构造pure_apm因子字典,并持久化 dict_pure_apm = { 'date': [date_label] * len(pure_symbol_lst), 'id': pure_symbol_lst, 'factorvalue': pure_apm_lst } pure_apm_db_file = os.path.join(factor_ct.FACTOR_DB.db_path, factor_ct.APM_CT.pure_apm_db_file) if save: Utils.factor_loading_persistent(pure_apm_db_file, calc_date.strftime('%Y%m%d'), dict_pure_apm) # 休息360秒 logging.info('Suspended for 360s.') time.sleep(360) return dict_apm
def _calc_synthetic_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的合成因子的载荷,并保存至因子数据库 Parameters -------- :param start_date: datetime-like, str 开始日期 :param end_date: datetime-like, str,默认None 结束日期,如果为None,则只计算start_date日期的因子载荷 :param month_end: bool,默认True 只计算月末时点的因子载荷,该参数只在end_date不为None时有效,并且不论end_date是否为None,都会计算第一天的因子载荷 :param save: 是否保存至因子数据库,默认为False :param kwargs: 'multi_proc': bool, True=采用多进程, False=采用单进程, 默认为False 'com_factors': list, 成分因子的类实例list :return: 因子载荷,DataFrame -------- 因子载荷,DataFrame 0: ID, 证券ID,为索引 1: factorvalue, 因子载荷 """ # 取得交易日序列 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) # 遍历交易日序列, 计算合成因子下各个成分因子的因子载荷 if 'multi_proc' not in kwargs: kwargs['multi_proc'] = False for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue # 计算各成分因子的因子载荷 # for com_factor in eval('risk_ct.' + cls.__name__.upper() + '_CT')['component']: # factor = eval(com_factor + '()') # factor.calc_factor_loading(start_date=calc_date, end_date=None, month_end=month_end, save=save, multi_proc=kwargs['multi_proc']) for com_factor in kwargs['com_factors']: com_factor.calc_factor_loading(start_date=calc_date, end_date=None, month_end=month_end, save=save, multi_proc=kwargs['multi_proc']) # 计算合成因子 synthetic_factor = pd.DataFrame() df_industry_classify = Utils.get_industry_classify() # 个股行业分类数据 for com_factor in eval('risk_ct.' + cls.__name__.upper() + '_CT')['component']: factor_path = os.path.join( factor_ct.FACTOR_DB.db_path, eval('risk_ct.' + com_factor + '_CT')['db_file']) factor_loading = Utils.read_factor_loading( factor_path, Utils.datetimelike_to_str(calc_date, dash=False)) factor_loading.drop(columns='date', inplace=True) factor_loading.rename(columns={'factorvalue': com_factor}, inplace=True) # 添加行业分类数据 factor_loading = pd.merge( left=factor_loading, right=df_industry_classify[['id', 'ind_code']], how='inner', on='id') # 取得含缺失值的因子载荷数据 missingdata_factor = factor_loading[ factor_loading[com_factor].isna()] # 删除factor_loading中的缺失值 factor_loading.dropna(axis='index', how='any', inplace=True) # 对factor_loading去极值、标准化 factor_loading = Utils.normalize_data(factor_loading, id='id', columns=com_factor, treat_outlier=True, weight='cap', calc_date=calc_date) # 把missingdata_factor中的缺失值替换为行业均值 ind_codes = set(missingdata_factor['ind_code']) ind_mean_factor = {} for ind_code in ind_codes: ind_mean_factor[ind_code] = factor_loading[ factor_loading['ind_code'] == ind_code][com_factor].mean() for idx, missingdata in missingdata_factor.iterrows(): missingdata_factor.loc[idx, com_factor] = ind_mean_factor[ missingdata['ind_code']] # 把missingdata_factor和factor_loading合并 factor_loading = pd.concat( [factor_loading, missingdata_factor]) # 删除ind_code列 factor_loading.drop(columns='ind_code', inplace=True) # merge成分因子 if synthetic_factor.empty: synthetic_factor = factor_loading else: synthetic_factor = pd.merge(left=synthetic_factor, right=factor_loading, how='inner', on='id') # 合成因子 synthetic_factor.set_index('id', inplace=True) weight = pd.Series( eval('risk_ct.' + cls.__name__.upper() + '_CT')['weight']) synthetic_factor = (synthetic_factor * weight).sum(axis=1) synthetic_factor.name = 'factorvalue' synthetic_factor.index.name = 'id' synthetic_factor = pd.DataFrame(synthetic_factor) synthetic_factor.reset_index(inplace=True) synthetic_factor['date'] = Utils.get_trading_days(start=calc_date, ndays=2)[1] # 保存synthetic_factor因子载荷 if save: Utils.factor_loading_persistent( cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), synthetic_factor.to_dict('list'), ['date', 'id', 'factorvalue'])
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷,并保存至因子数据库 Parameters -------- :param start_date: datetime-like, str 开始日期,格式:YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期,如果为None,则只计算start_date日期的因子载荷,格式:YYYY-MM-DD or YYYYMMDD :param month_end: bool,默认True 如果为True,则只计算月末时点的因子载荷 :param save: bool,默认False 是否保存至因子数据库 :param kwargs['synthetic_factor']: bool, 默认为False 是否计算合成因子 :return: 因子载荷,DataFrame -------- 因子载荷,DataFrame 0. date: 日期 1. id: 证券symbol 2. m0: 隔夜时段动量 3. m1: 第一个小时动量 4. m2: 第二个小时动量 5. m3: 第三个小时动量 6. m4: 第四个小时动量 7. m_normal: 传统动量 """ # 取得交易日序列及股票基本信息表 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列,计算日内动量因子值 dict_intraday_momentum = None for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue logging.info( '[%s] calc synthetic intraday momentum factor loading.' % Utils.datetimelike_to_str(calc_date)) if 'synthetic_factor' in kwargs and kwargs[ 'synthetic_factor']: # 计算日内合成动量因子 dict_intraday_momentum = { 'date': [], 'id': [], 'factorvalue': [] } # 读取日内个时段动量因子值 df_factor_loading = Utils.read_factor_loading( cls._db_file, Utils.datetimelike_to_str(calc_date, False)) if df_factor_loading.shape[0] <= 0: logging.info( "[%s] It doesn't exist intraday momentum factor loading." % Utils.datetimelike_to_str(calc_date)) return df_factor_loading.fillna(0, inplace=True) # 读取因子最优权重 factor_weight = cls.get_factor_weight(calc_date) if factor_weight is None: logging.info("[%s] It doesn't exist factor weight.") return # 计算合成动量因子 arr_factor_loading = np.array( df_factor_loading[['m0', 'm1', 'm2', 'm3', 'm4']]) arr_factor_weight = np.array( factor_weight.drop('date')).reshape((5, 1)) arr_synthetic_factor = np.dot(arr_factor_loading, arr_factor_weight) # arr_synthetic_factor = np.around(arr_synthetic_factor, 6) dict_intraday_momentum['date'] = list( df_factor_loading['date']) dict_intraday_momentum['id'] = list(df_factor_loading['id']) dict_intraday_momentum['factorvalue'] = list( arr_synthetic_factor.astype(float).round(6).reshape( (arr_synthetic_factor.shape[0], ))) # 保存合成因子 if save: synthetic_db_file = os.path.join( factor_ct.FACTOR_DB.db_path, factor_ct.INTRADAYMOMENTUM_CT.synthetic_db_file) Utils.factor_loading_persistent( synthetic_db_file, Utils.datetimelike_to_str(calc_date, False), dict_intraday_momentum) else: # 计算日内各时段动量因子 dict_intraday_momentum = { 'date': [], 'id': [], 'm0': [], 'm1': [], 'm2': [], 'm3': [], 'm4': [], 'm_normal': [] } # 遍历个股,计算个股日内动量值 s = (calc_date - datetime.timedelta(days=90)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] # 采用单进程进行计算 # for _, stock_info in stock_basics.iterrows(): # momentum_data = cls._calc_factor_loading(stock_info.symbol, calc_date) # if momentum_data is not None: # logging.info("[%s] %s's intraday momentum = (%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f)" % (calc_date.strftime('%Y-%m-%d'),stock_info.symbol, momentum_data.m0, momentum_data.m1, momentum_data.m2, momentum_data.m3, momentum_data.m4, momentum_data.m_normal)) # dict_intraday_momentum['id'].append(Utils.code_to_symbol(stock_info.symbol)) # dict_intraday_momentum['m0'].append(round(momentum_data.m0, 6)) # dict_intraday_momentum['m1'].append(round(momentum_data.m1, 6)) # dict_intraday_momentum['m2'].append(round(momentum_data.m2, 6)) # dict_intraday_momentum['m3'].append(round(momentum_data.m3, 6)) # dict_intraday_momentum['m4'].append(round(momentum_data.m4, 6)) # dict_intraday_momentum['m_normal'].append(round(momentum_data.m_normal, 6)) # 采用多进程并行计算日内动量因子载荷 q = Manager().Queue() # 队列,用于进程间通信,存储每个进程计算的因子载荷 p = Pool(4) # 进程池,最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=( stock_info.symbol, calc_date, q, )) p.close() p.join() while not q.empty(): momentum_data = q.get(True) dict_intraday_momentum['id'].append(momentum_data[0]) dict_intraday_momentum['m0'].append( round(momentum_data[1], 6)) dict_intraday_momentum['m1'].append( round(momentum_data[2], 6)) dict_intraday_momentum['m2'].append( round(momentum_data[3], 6)) dict_intraday_momentum['m3'].append( round(momentum_data[4], 6)) dict_intraday_momentum['m4'].append( round(momentum_data[5], 6)) dict_intraday_momentum['m_normal'].append( round(momentum_data[6], 6)) date_label = Utils.get_trading_days(calc_date, ndays=2)[1] dict_intraday_momentum['date'] = [date_label] * len( dict_intraday_momentum['id']) # 保存因子载荷至因子数据库 if save: Utils.factor_loading_persistent( cls._db_file, calc_date.strftime('%Y%m%d'), dict_intraday_momentum) # 休息360秒 logging.info('Suspending for 360s.') time.sleep(360) return dict_intraday_momentum
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷, 并保存至因子数据库 Parameters: -------- :param start_date: datetime-like, str 开始日期, 格式: YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期, 如果为None, 则只计算start_date日期的因子载荷, 格式: YYYY-MM-DD or YYYYMMDD :param month_end: bool, 默认为True 如果为True, 则只计算月末时点的因子载荷 :param save: bool, 默认True 是否保存至因子数据库 :param kwargs: :return: dict 因子载荷 """ # 取得交易日序列及股票基本信息表 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) # all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列, 计算筹码分布因子载荷 dict_beta = {} dict_hsigma = {} for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue logging.info('[%s] Calc BETA factor loading.' % Utils.datetimelike_to_str(calc_date)) # 遍历个股, 计算个股BETA因子值 # s = (calc_date - datetime.timedelta(days=risk_ct.DBETA_CT.listed_days)).strftime('%Y%m%d') # stock_basics = all_stock_basics[all_stock_basics.list_date < s] s = calc_date - datetime.timedelta(days=risk_ct.DBETA_CT.listed_days) stock_basics = Utils.get_stock_basics(s, False) ids = [] # 个股代码list betas = [] # BETA因子值 hsigmas = [] # HSIGMA因子值 if 'multi_proc' not in kwargs: kwargs['multi_proc'] = False if not kwargs['multi_proc']: # 采用单进程计算BETA因子和HSIGMA因子值, for _, stock_info in stock_basics.iterrows(): logging.debug("[%s] Calc %s's BETA and HSIGMA factor data." % (calc_date.strftime('%Y-%m-%d'), stock_info.symbol)) beta_data = cls._calc_factor_loading(stock_info.symbol, calc_date) if beta_data is None: ids.append(Utils.code_to_symbol(stock_info.symbol)) betas.append(np.nan) hsigmas.append(np.nan) else: ids.append(beta_data['code']) betas.append(beta_data['beta']) hsigmas.append(beta_data['hsigma']) else: # 采用多进程并行计算BETA因子和HSIGMA因子值 q = Manager().Queue() # 队列, 用于进程间通信, 存储每个进程计算的因子载荷 p = Pool(SETTINGS.CONCURRENCY_KERNEL_NUM) # 进程池, 最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=(stock_info.symbol, calc_date, q,)) p.close() p.join() while not q.empty(): beta_data = q.get(True) ids.append(beta_data['code']) betas.append(beta_data['beta']) hsigmas.append(beta_data['hsigma']) date_label = Utils.get_trading_days(calc_date, ndays=2)[1] dict_beta = {'date': [date_label]*len(ids), 'id': ids, 'factorvalue': betas} dict_hsigma = {'date': [date_label]*len(ids), 'id': ids, 'factorvalue': hsigmas} if save: Utils.factor_loading_persistent(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), dict_beta, ['date', 'id', 'factorvalue']) hsigma_path = os.path.join(factor_ct.FACTOR_DB.db_path, risk_ct.HSIGMA_CT.db_file) Utils.factor_loading_persistent(hsigma_path, Utils.datetimelike_to_str(calc_date, dash=False), dict_hsigma, ['date', 'id', 'factorvalue']) # 休息180秒 # logging.info('Suspending for 180s.') # time.sleep(180) return dict_beta
def _calc_MVPFP(factor_name, start_date, end_date=None, month_end=True, save=False): """ 构建目标因子的最小波动纯因子组合(Minimum Volatility Pure Factor Portfolio, MVPFP) Parameters: -------- :param factor_name: str alpha因子名称, e.g: SmartMoney :param start_date: datetime-like, str 开始日期, e.g: YYYY-MM-DD, YYYYMMDD :param end_date: datetime-like, str, 默认为None 结束日期, e.g: YYYY-MM-DD, YYYYMMDD :param month_end: bool, 默认为True 是否只计算月末日期的因子载荷 :param save: bool, 默认为False 是否保存计算结果 :return: CWeightHolding类 最小波动纯因子组合权重数据 -------- 具体优化算法:暴露1单位目标因子敞口, 同时保持其余所有风险因子的敞口为0, 并具有最小预期波动率的组合 Min: W'VW s.t. W'X_beta = 0 W'x_target = 1 其中: W: 最小波动纯因子组合对应的权重 V: 个股协方差矩阵 X_beta: 个股风格因子载荷矩阵 x_target: 个股目标因子载荷向量 """ start_date = Utils.to_date(start_date) if end_date is None: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) else: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) CRiskModel = Barra() mvpfp_holding = CWeightHolding() for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue # 取得/计算calc_date的个股协方差矩阵数据 stock_codes, arr_stocks_covmat = CRiskModel.calc_stocks_covmat( calc_date) # 取得个股风格因子载荷矩阵数据 df_stylefactor_loading = CRiskModel.get_StyleFactorloading_matrix( calc_date) # df_stylefactor_loading.set_index('code', inplace=True) # df_stylefactor_loading = df_stylefactor_loading.loc[stock_codes] # 按个股顺序重新排列 # arr_stylefactor_loading = np.array(df_stylefactor_loading) # 取得个股目标因子载荷向量数据(正交化后的因子载荷) df_targetfactor_loading = _get_factorloading( factor_name, calc_date, alphafactor_ct.FACTORLOADING_TYPE['ORTHOGONALIZED']) df_targetfactor_loading.drop(columns='date', inplace=True) df_targetfactor_loading.rename(columns={ 'id': 'code', 'factorvalue': factor_name }, inplace=True) df_factorloading = pd.merge(left=df_stylefactor_loading, right=df_targetfactor_loading, how='inner', on='code') df_factorloading.set_index('code', inplace=True) df_stylefactor_loading = df_factorloading.loc[ stock_codes, riskfactor_ct.STYLE_RISK_FACTORS] arr_stylefactor_laoding = np.array(df_stylefactor_loading) df_targetfactor_loading = df_factorloading.loc[stock_codes, factor_name] arr_targetfactor_loading = np.array(df_targetfactor_loading) # 优化计算最小波动纯因子组合权重 V = arr_stocks_covmat X_beta = arr_stylefactor_laoding x_target = arr_targetfactor_loading N = len(stock_codes) w = cvx.Variable((N, 1)) risk = cvx.quad_form(w, V) constraints = [ cvx.matmul(w.T, X_beta) == 0, cvx.matmul(w.T, x_target) == 1 ] prob = cvx.Problem(cvx.Minimize(risk), constraints) prob.solve() if prob.status == cvx.OPTIMAL: datelabel = Utils.datetimelike_to_str(calc_date, dash=False) df_holding = pd.DataFrame({ 'date': [datelabel] * len(stock_codes), 'code': stock_codes, 'weight': w.value }) mvpfp_holding.from_dataframe(df_holding) if save: holding_path = os.path.join( SETTINGS.FACTOR_DB_PATH, eval('alphafactor_ct.' + factor_name.upper() + '.CT')['db_file'], 'mvpfp', '{}_{}.csv'.format(factor_name, datelabel)) mvpfp_holding.save_data(holding_path) else: raise cvx.SolverError( "%s优化计算%s最小纯因子组合失败。" % (Utils.datetimelike_to_str(calc_date), factor_name)) return mvpfp_holding
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷, 并保存至因子数据库 Parameters: -------- :param start_date: datetime-like, str 开始日期, 格式: YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期, 如果为None, 则只计算start_date日期的因子载荷, 格式: YYYY-MM-DD or YYYYMMDD :param month_end: bool, 默认为True 如果为True, 则只计算月末时点的因子载荷 :param save: bool, 默认为True 是否保存至因子数据库 :param kwargs: 'multi_proc': bool, True=采用多进程, False=采用单进程, 默认为False :return: dict 因子载荷数据 """ # 取得交易日序列及股票基本信息表 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列, 计算LIQUIDITY因子载荷 dict_raw_liquidity = None for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue dict_stom = None dict_stoq = None dict_stoa = None dict_raw_liquidity = None logging.info('[%s] Calc LIQUIDITY factor loading.' % Utils.datetimelike_to_str(calc_date)) # 遍历个股,计算个股LIQUIDITY因子值 s = (calc_date - datetime.timedelta( days=risk_ct.LIQUID_CT.listed_days)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] ids = [] stoms = [] stoqs = [] stoas = [] raw_liquidities = [] if 'multi_proc' not in kwargs: kwargs['multi_proc'] = False if not kwargs['multi_proc']: # 采用单进程计算LIQUIDITY因子值 for _, stock_info in stock_basics.iterrows(): logging.info("[%s] Calc %s's LIQUIDITY factor loading." % (Utils.datetimelike_to_str( calc_date, dash=True), stock_info.symbol)) liquidity_data = cls._calc_factor_loading( stock_info.symbol, calc_date) if liquidity_data is not None: ids.append(liquidity_data['code']) stoms.append(liquidity_data['stom']) stoqs.append(liquidity_data['stoq']) stoas.append(liquidity_data['stoa']) raw_liquidities.append(liquidity_data['liquidity']) else: # 采用多进程计算LIQUIDITY因子值 q = Manager().Queue() p = Pool(4) for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=( stock_info.symbol, calc_date, q, )) p.close() p.join() while not q.empty(): liquidity_data = q.get(True) ids.append(liquidity_data['code']) stoms.append(liquidity_data['stom']) stoqs.append(liquidity_data['stoq']) stoas.append(liquidity_data['stoa']) raw_liquidities.append(liquidity_data['liquidity']) date_label = Utils.get_trading_days(start=calc_date, ndays=2)[1] dict_stom = dict({ 'date': [date_label] * len(ids), 'id': ids, 'factorvalue': stoms }) dict_stoq = dict({ 'date': [date_label] * len(ids), 'id': ids, 'factorvalue': stoqs }) dict_stoa = dict({ 'date': [date_label] * len(ids), 'id': ids, 'factorvalue': stoas }) dict_raw_liquidity = dict({ 'date': [date_label] * len(ids), 'id': ids, 'factorvalue': raw_liquidities }) # 读取Size因子值, 将流动性因子与Size因子正交化 size_factor_path = os.path.join(factor_ct.FACTOR_DB.db_path, risk_ct.SIZE_CT.db_file) df_size = Utils.read_factor_loading( size_factor_path, Utils.datetimelike_to_str(calc_date, dash=False)) df_size.drop(columns='date', inplace=True) df_size.rename(columns={'factorvalue': 'size'}, inplace=True) df_liquidity = pd.DataFrame( dict({ 'id': ids, 'liquidity': raw_liquidities })) df_liquidity = pd.merge(left=df_liquidity, right=df_size, how='inner', on='id') arr_liquidity = Utils.normalize_data( Utils.clean_extreme_value( np.array(df_liquidity['liquidity']).reshape( (len(df_liquidity), 1)))) arr_size = Utils.normalize_data( Utils.clean_extreme_value( np.array(df_liquidity['size']).reshape( (len(df_liquidity), 1)))) model = sm.OLS(arr_liquidity, arr_size) results = model.fit() df_liquidity['liquidity'] = results.resid df_liquidity.drop(columns='size', inplace=True) df_liquidity.rename(columns={'liquidity': 'factorvalue'}, inplace=True) df_liquidity['date'] = date_label # 保存因子载荷 if save: str_date = Utils.datetimelike_to_str(calc_date, dash=False) factor_header = ['date', 'id', 'factorvalue'] Utils.factor_loading_persistent(cls._db_file, 'stom_{}'.format(str_date), dict_stom, factor_header) Utils.factor_loading_persistent(cls._db_file, 'stoq_{}'.format(str_date), dict_stoq, factor_header) Utils.factor_loading_persistent(cls._db_file, 'stoa_{}'.format(str_date), dict_stoa, factor_header) Utils.factor_loading_persistent( cls._db_file, 'rawliquidity_{}'.format(str_date), dict_raw_liquidity, factor_header) Utils.factor_loading_persistent(cls._db_file, str_date, df_liquidity.to_dict('list'), factor_header) # 暂停180秒 logging.info('Suspending for 180s.') time.sleep(180) return dict_raw_liquidity
def calc_factor_loading1(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷, 并保存至因子数据库 Parameters: -------- :param start_date: datetime-like or str 开始日期, 格式: YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期, 如果为None, 则只计算start_date日期的因子载荷, 格式:YYYY-MM-DD or YYYYMMDD :param month_end: bool, 默认True 如果为True, 则只计算月末时点的因子载荷 :param save: bool, 默认True 是否保存至因子数据库 :param kwargs: :return: dict 因子载荷 -------- """ # 取得交易日序列及股票基本信息表 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列, 计算筹码分布因子载荷 dict_cyq = None for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue logging.info('[%s] Calc CYQ factor loading.' % Utils.datetimelike_to_str(calc_date)) # 遍历个股, 计算个股筹码分布因子值 df_proxies = DataFrame() s = (calc_date - datetime.timedelta(days=365)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] trading_day = Utils.get_trading_days(calc_date, ndays=2)[1] # 采用单进程计算筹码因子分布的代理变量 # for _, stock_info in stock_basics.iterrows(): # cyq_proxies = cls._calc_factor_loading(stock_info.symbol, calc_date) # if cyq_proxies is not None: # logging.info("[%s] %s's cyq proxies = (%0.4f,%0.4f,%0.4f,%0.4f,%0.4f)" % (calc_date.strftime('%Y-%m-%d'), stock_info.symbol, cyq_proxies['arc'], cyq_proxies['vrc'], cyq_proxies['src'], cyq_proxies['krc'], cyq_proxies['next_ret'])) # # cyq_proxies['date'] = trading_day # cyq_proxies['id'] = Utils.code_to_symbol(stock_info.symbol) # df_proxies = df_proxies.append(cyq_proxies, ignore_index=True) # 采用多进程进行并行计算筹码分布因子的代理变量 q = Manager().Queue() # 队列, 用于进程间通信, 存储每个进程计算的因子载荷 p = Pool(4) # 进程池, 最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=( stock_info.symbol, calc_date, q, )) p.close() p.join() while not q.empty(): cyq_proxies = q.get(True) # cyq_proxies['date'] = trading_day df_proxies = df_proxies.append(cyq_proxies, ignore_index=True) # 保存筹码分布代理变量数据 df_proxies['date'] = trading_day proxies_file_path = cls._db_proxies_path + '_%s.csv' % Utils.datetimelike_to_str( calc_date, dash=False) df_proxies.to_csv( proxies_file_path, index=False, columns=['date', 'id', 'arc', 'vrc', 'src', 'krc', 'next_ret']) # 导入筹码分布因子的代理变量数据 # cyq_proxies_path = cls._db_proxies_path + '_%s.csv' % Utils.datetimelike_to_str(calc_date, dash=False) # df_proxies = pd.read_csv(cyq_proxies_path, header=0) # 计算marc, 代理变量权重及筹码分布因子载荷 marc = df_proxies['arc'].median() proxies_weight_file = Path(factor_ct.FACTOR_DB.db_path, factor_ct.CYQ_CT.proxies_weight_file) if proxies_weight_file.exists(): df_proxies_weight = pd.read_csv(proxies_weight_file, header=0, parse_dates=[0]) df_proxies_weight = df_proxies_weight[ df_proxies_weight.date < calc_date].tail(24) if len(df_proxies_weight) < 24: with open(proxies_weight_file, 'a', newline='') as f: csv_writer = csv.writer(f) csv_writer.writerow([ calc_date.strftime('%Y-%m-%d'), marc, 0, 0, 0, 0, 0 ]) else: df_proxies_data = DataFrame() if marc > 0: df_proxies_weight = df_proxies_weight[ df_proxies_weight.marc > 0] elif marc < 0: df_proxies_weight = df_proxies_weight[ df_proxies_weight.marc < 0] for _, weight_info in df_proxies_weight.iterrows(): proxies_file_path = cls._db_proxies_path + '_%s.csv' % Utils.datetimelike_to_str( weight_info['date'], False) df_proxies_data = df_proxies_data.append( pd.read_csv(proxies_file_path, header=0), ignore_index=True) next_ret = np.array(df_proxies_data['next_ret']) cyq_data = np.array( df_proxies_data[['arc', 'vrc', 'src', 'krc']]) cyq_data = sm.add_constant(cyq_data) cyq_model = sm.OLS(next_ret, cyq_data) cyq_result = cyq_model.fit() cyq_weights = np.around(cyq_result.params, 6) with open(proxies_weight_file, 'a', newline='') as f: csv_writer = csv.writer(f) csv_writer.writerow([ calc_date.strftime('%Y-%m-%d'), marc, cyq_weights[0], cyq_weights[1], cyq_weights[2], cyq_weights[3], cyq_weights[4] ]) # 计算筹码分布因子载荷 arr_proxies = np.array( df_proxies[['arc', 'vrc', 'src', 'krc']]) arr_weight = np.array([ cyq_weights[1], cyq_weights[2], cyq_weights[3], cyq_weights[4] ]).reshape((4, 1)) intercept = cyq_weights[0] arr_cyq = np.around( np.dot(arr_proxies, arr_weight) + intercept, 6) dict_cyq = { 'date': list(df_proxies['date']), 'id': list(df_proxies['id']), 'factorvalue': list(arr_cyq.reshape((len(arr_cyq), ))) } # 保存因子载荷至因子数据库 if save: Utils.factor_loading_persistent( cls._db_file, calc_date.strftime('%Y%m%d'), dict_cyq, columns=['date', 'id', 'factorvalue']) else: with open(proxies_weight_file, 'w', newline='') as f: csv_writer = csv.writer(f) csv_writer.writerow([ 'date', 'marc', 'intcpt', 'arc_w', 'vrc_w', 'src_w', 'krc_w' ]) csv_writer.writerow( [calc_date.strftime('%Y-%m-%d'), marc, 0, 0, 0, 0, 0]) # 休息300秒 logging.info('Suspending for 200s.') time.sleep(200)
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷, 并保存至因子数据库 Parameters: -------- :param start_date: datetime-like or str 开始日期, 格式: YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期, 如果为None, 则只计算start_date日期的因子载荷, 格式:YYYY-MM-DD or YYYYMMDD :param month_end: bool, 默认True 如果为True, 则只计算月末时点的因子载荷 :param save: bool, 默认True 是否保存至因子数据库 :param kwargs: :return: dict 因子载荷 -------- """ # 取得交易日序列及股票基本信息表 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列, 计算筹码分布因子载荷 dict_cyq = {} for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue logging.info('[%s] Calc CYQ factor loading.' % Utils.datetimelike_to_str(calc_date)) # 遍历个股, 计算个股筹码分布因子值 s = (calc_date - datetime.timedelta(days=180)).strftime('%Y%m%d') stock_basics = all_stock_basics[all_stock_basics.list_date < s] secu_cyq_path = Path( factor_ct.FACTOR_DB.db_path, factor_ct.CYQ_CT.db_file, 'secu_cyq/%s' % calc_date.strftime('%Y-%m-%d')) if not secu_cyq_path.exists(): secu_cyq_path.mkdir() ids = [] rps = [] # 采用单进程计算筹码分布数据, 及当前价格的相对位置(=当前价格-平均成本)/平均成本 # for _, stock_info in stock_basics.iterrows(): # logging.info("[%s] Calc %s's cyq data." % (calc_date.strftime('%Y-%m-%d'), stock_info.symbol)) # secu_cyq = cls._calc_factor_loading(stock_info.symbol, calc_date) # if secu_cyq is not None: # secu_code, secu_close, cyq_data = secu_cyq # # 保存个股的筹码分布数据 # cyq_data.to_csv(Path(secu_cyq_path, '%s.csv' % secu_code), header=True) # # 计算当前价格的相对位置 # avg_cyq = np.sum(np.array(cyq_data.index) * np.array(cyq_data.values)) # relative_position = round((secu_close - avg_cyq) / avg_cyq, 4) # ids.append(secu_code) # rps.append(relative_position) # 采用多进程进行并行计算筹码分布数据, 及当前价格的相对位置(=当前价格-平均成本)/平均成本 q = Manager().Queue() # 队列, 用于进程间通信, 存储每个进程计算的因子载荷 p = Pool(4) # 进程池, 最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=( stock_info.symbol, calc_date, q, )) p.close() p.join() while not q.empty(): secu_cyq = q.get(True) secu_code, secu_close, cyq_data = secu_cyq # 保存个股的筹码分布数据 cyq_data.to_csv(Path(secu_cyq_path, '%s.csv' % secu_code), header=True) # 计算当前价格的相对位置 avg_cyq = np.sum( np.array(cyq_data.index) * np.array(cyq_data.values)) relative_position = round((secu_close - avg_cyq) / avg_cyq, 4) ids.append(secu_code) rps.append(relative_position) date_label = Utils.get_trading_days(calc_date, ndays=2)[1] dict_cyq = { 'date': [date_label] * len(ids), 'id': ids, 'factorvalue': rps } if save: cyq_data_path = os.path.join(factor_ct.FACTOR_DB.db_path, factor_ct.CYQ_CT.db_file, factor_ct.CYQ_CT.CYQ_rp_file) Utils.factor_loading_persistent( cyq_data_path, Utils.datetimelike_to_str(calc_date, dash=False), dict_cyq, ['date', 'id', 'factorvalue']) # 休息90秒 logging.info('Suspending for 100s.') time.sleep(100) return dict_cyq
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷,并保存至因子数据库 Parameters -------- :param start_date: datetime-like, str 开始日期 :param end_date: datetime-like, str,默认None 结束日期,如果为None,则只计算start_date日期的因子载荷 :param month_end: bool,默认True 只计算月末时点的因子载荷 :param save: 是否保存至因子数据库,默认为False :param kwargs: 'multi_proc': bool, True=采用多进程并行计算, False=采用单进程计算, 默认为False :return: 因子载荷,DataFrame -------- 因子载荷,DataFrame 0. date, 日期, 为计算日期的下一个交易日 1: id, 证券代码 2: factorvalue, 因子载荷 如果end_date=None,返回start_date对应的因子载荷数据 如果end_date!=None,返回最后一天的对应的因子载荷数据 如果没有计算数据,返回None """ # 0.取得交易日序列 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) # 取得样本个股信息 # all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列,计算SMartQ因子载荷 dict_factor = None for calc_date in trading_days_series: dict_factor = {'id': [], 'factorvalue': []} if month_end and (not Utils.is_month_end(calc_date)): continue # 1.获取用于读取分钟行情的交易日列表(过去30天的交易日列表,降序排列) # trading_days = _get_trading_days(calc_date, 30) # trading_days = Utils.get_trading_days(end=calc_date, ndays=30, ascending=False) # 2.取得样本个股信息 # stock_basics = ts.get_stock_basics() s = (calc_date - datetime.timedelta(days=90)).strftime('%Y%m%d') stock_basics = Utils.get_stock_basics(s) # 3.遍历样本个股代码,计算Smart_Q因子载荷值 dict_factor = {'date': None, 'id': [], 'factorvalue': []} if 'multi_proc' not in kwargs: kwargs['multi_proc'] = False if not kwargs['multi_proc']: # 采用单进程进行计算 for _, stock_info in stock_basics.iterrows(): # code = '%s%s' % ('SH' if code[:2] == '60' else 'SZ', code) factor_loading = cls._calc_factor_loading( stock_info.symbol, calc_date) print( "[%s]Calculating %s's SmartMoney factor loading = %.4f." % (calc_date.strftime('%Y-%m-%d'), stock_info.symbol, -1.0 if factor_loading is None else factor_loading)) if factor_loading is not None: # df_factor.ix[code, 'factorvalue'] = factor_loading dict_factor['id'].append( Utils.code_to_symbol(stock_info.symbol)) dict_factor['factorvalue'].append(factor_loading) else: # 采用多进程并行计算SmartQ因子载荷 q = Manager().Queue() # 队列,用于进程间通信,存储每个进程计算的因子载荷值 p = Pool(4) # 进程池,最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=( stock_info.symbol, calc_date, q, )) p.close() p.join() while not q.empty(): smart_q = q.get(True) dict_factor['id'].append(smart_q[0]) dict_factor['factorvalue'].append(smart_q[1]) date_label = Utils.get_trading_days(calc_date, ndays=2)[1] dict_factor['date'] = [date_label] * len(dict_factor['id']) # 4.计算去极值标准化后的因子载荷 df_std_factor = Utils.normalize_data(pd.DataFrame(dict_factor), columns='factorvalue', treat_outlier=True, weight='eq') # 5.保存因子载荷至因子数据库 if save: # Utils.factor_loading_persistent(cls._db_file, calc_date.strftime('%Y%m%d'), dict_factor) cls._save_factor_loading(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), dict_factor, 'SmartMoney', factor_type='raw', columns=['date', 'id', 'factorvalue']) cls._save_factor_loading(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), df_std_factor, 'SmartMoney', factor_type='standardized', columns=['date', 'id', 'factorvalue']) # 休息300秒 logging.info('Suspending for 360s.') time.sleep(360) return dict_factor
def calc_factor_loading(cls, start_date, end_date=None, month_end=True, save=False, **kwargs): """ 计算指定日期的样本个股的因子载荷,并保存至因子数据库 Parameters -------- :param start_date: datetime-like, str 开始日期,格式:YYYY-MM-DD or YYYYMMDD :param end_date: datetime-like, str 结束日期,如果为None,则只计算start_date日期的因子载荷,格式:YYYY-MM-DD or YYYYMMDD :param month_end: bool,默认True 如果为True,则只计算月末时点的因子载荷 :param save: bool,默认False 是否保存至因子数据库 :param kwargs: 'multi_proc': bool, True=采用多进程并行计算, False=采用单进程计算, 默认为False :return: 因子载荷,DataFrame -------- 因子载荷,DataFrame 0. date: 日期 1. id: 证券symbol 2. m0: 隔夜时段动量 3. m1: 第一个小时动量 4. m2: 第二个小时动量 5. m3: 第三个小时动量 6. m4: 第四个小时动量 7. m_normal: 传统动量 """ # 取得交易日序列及股票基本信息表 start_date = Utils.to_date(start_date) if end_date is not None: end_date = Utils.to_date(end_date) trading_days_series = Utils.get_trading_days(start=start_date, end=end_date) else: trading_days_series = Utils.get_trading_days(end=start_date, ndays=1) # all_stock_basics = CDataHandler.DataApi.get_secu_basics() # 遍历交易日序列,计算日内动量因子值 dict_intraday_momentum = None for calc_date in trading_days_series: if month_end and (not Utils.is_month_end(calc_date)): continue # 计算日内各时段动量因子 dict_intraday_momentum = {'date': [], 'id': [], 'm0': [], 'm1': [], 'm2': [], 'm3': [], 'm4': [], 'm_normal': []} # 遍历个股,计算个股日内动量值 s = (calc_date - datetime.timedelta(days=90)).strftime('%Y%m%d') stock_basics = Utils.get_stock_basics(s) if 'multi_proc' not in kwargs: kwargs['multi_proc'] = False if not kwargs['multi_proc']: # 采用单进程进行计算 for _, stock_info in stock_basics.iterrows(): momentum_data = cls._calc_factor_loading(stock_info.symbol, calc_date) if momentum_data is not None: logging.info("[%s] %s's intraday momentum = (%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f)" % (calc_date.strftime('%Y-%m-%d'),stock_info.symbol, momentum_data.m0, momentum_data.m1, momentum_data.m2, momentum_data.m3, momentum_data.m4, momentum_data.m_normal)) dict_intraday_momentum['id'].append(Utils.code_to_symbol(stock_info.symbol)) dict_intraday_momentum['m0'].append(round(momentum_data.m0, 6)) dict_intraday_momentum['m1'].append(round(momentum_data.m1, 6)) dict_intraday_momentum['m2'].append(round(momentum_data.m2, 6)) dict_intraday_momentum['m3'].append(round(momentum_data.m3, 6)) dict_intraday_momentum['m4'].append(round(momentum_data.m4, 6)) dict_intraday_momentum['m_normal'].append(round(momentum_data.m_normal, 6)) else: # 采用多进程并行计算日内动量因子载荷 q = Manager().Queue() # 队列,用于进程间通信,存储每个进程计算的因子载荷 p = Pool(4) # 进程池,最多同时开启4个进程 for _, stock_info in stock_basics.iterrows(): p.apply_async(cls._calc_factor_loading_proc, args=(stock_info.symbol, calc_date, q,)) p.close() p.join() while not q.empty(): momentum_data = q.get(True) dict_intraday_momentum['id'].append(momentum_data[0]) dict_intraday_momentum['m0'].append(round(momentum_data[1], 6)) dict_intraday_momentum['m1'].append(round(momentum_data[2], 6)) dict_intraday_momentum['m2'].append(round(momentum_data[3], 6)) dict_intraday_momentum['m3'].append(round(momentum_data[4], 6)) dict_intraday_momentum['m4'].append(round(momentum_data[5], 6)) dict_intraday_momentum['m_normal'].append(round(momentum_data[6], 6)) date_label = Utils.get_trading_days(calc_date, ndays=2)[1] dict_intraday_momentum['date'] = [date_label] * len(dict_intraday_momentum['id']) # 保存因子载荷至因子数据库 if save: # Utils.factor_loading_persistent(cls._db_file, calc_date.strftime('%Y%m%d'), dict_intraday_momentum) cls._save_factor_loading(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), dict_intraday_momentum, 'periodmomentum', factor_type='raw') # 计算日内各时段动量因子的Rank IC值向量, 并保存 cls._calc_periodmomentum_ic(calc_date, 'month') # 计算最优化权重 if alphafactor_ct.INTRADAYMOMENTUM_CT['optimized']: cls._optimize_periodmomentum_weight(calc_date) # 计算合成日内动量因子 if alphafactor_ct.INTRADAYMOMENTUM_CT['synthesized']: logging.info('[%s] calc synthetic intraday momentum factor loading.' % Utils.datetimelike_to_str(calc_date)) dict_intraday_momentum = {'date': [], 'id': [], 'factorvalue': []} # 读取日内个时段动量因子值 # period_momentum_path = os.path.join(SETTINGS.FACTOR_DB_PATH, alphafactor_ct.INTRADAYMOMENTUM_CT.db_file, 'raw/periodmomentum') # df_factor_loading = Utils.read_factor_loading(period_momentum_path, Utils.datetimelike_to_str(calc_date, False)) df_factor_loading = cls._get_factor_loading(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), factor_name='periodmomentum', factor_type='raw', drop_na=False) if df_factor_loading.shape[0] <= 0: logging.info("[%s] It doesn't exist intraday momentum factor loading." % Utils.datetimelike_to_str(calc_date)) return df_factor_loading.fillna(0, inplace=True) # 读取因子最优权重 factor_weight = cls._get_factor_weight(calc_date) if factor_weight is None: logging.info("[%s] It doesn't exist factor weight.") return # 计算合成动量因子, 合成之前先对日内各时段动量因子进行去极值和标准化处理 arr_factor_loading = np.array(df_factor_loading[['m0', 'm1', 'm2', 'm3', 'm4']]) arr_factor_loading = Utils.normalize_data(arr_factor_loading, treat_outlier=True) arr_factor_weight = np.array(factor_weight.drop('date')).reshape((5, 1)) arr_synthetic_factor = np.dot(arr_factor_loading, arr_factor_weight) dict_intraday_momentum['date'] = list(df_factor_loading['date']) dict_intraday_momentum['id'] = list(df_factor_loading['id']) dict_intraday_momentum['factorvalue'] = list(arr_synthetic_factor.astype(float).round(6).reshape((arr_synthetic_factor.shape[0],))) # 标准化合成动量因子 df_std_intradaymonmentum = Utils.normalize_data(pd.DataFrame(dict_intraday_momentum), columns='factorvalue', treat_outlier=True, weight='eq') # 保存合成因子 if save: # Utils.factor_loading_persistent(synthetic_db_file, Utils.datetimelike_to_str(calc_date, False), dict_intraday_momentum) cls._save_factor_loading(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), dict_intraday_momentum, 'IntradayMomentum', factor_type='raw', columns=['date', 'id', 'factorvalue']) cls._save_factor_loading(cls._db_file, Utils.datetimelike_to_str(calc_date, dash=False), df_std_intradaymonmentum, 'IntradayMomentum', factor_type='standardized', columns=['date', 'id', 'factorvalue']) # 休息360秒 logging.info('Suspending for 360s.') time.sleep(360) return dict_intraday_momentum