end_date,
year=0,
month=6)
else:
where = Now_Index + " = 1"
components_dict = db_interaction.get_data_commonly(
"daily_index_components", ["stock_id"], ["curr_date"],
one_to_one=0,
where=where)
'''
***从因子表中获取我们需要的几列数据***
'''
descriptor_data_dict = db_interaction.get_daily_data_dict_1_key(
start_date,
end_date,
Factor_Table_Name,
change_factor_list + keep_value_list,
date_for_key=1,
to_df=1)
'''
***根据Index篮子做OLS***
'''
output_data_dict = {}
for date in daily_date_list:
print date, 'calculating size residual factors...'
#筛选本日成分股的数据
temp_mi_df = pd.DataFrame(components_dict[date], columns=["stock_id"])
this_date_components_df = descriptor_data_dict[date].loc[
descriptor_data_dict[date].index.intersection(
temp_mi_df.set_index('stock_id').index)]
#先写入那些保持不变的量
SHIBOR_dict = xyk_common_wind_db_interaction.get_IBOR_dict(
"SHIBOR", "SHIBORON.IR")
index_return_dict = db_interaction.get_data_commonly(
"daily_index_performance", ["cap_weighted_return"],
["index_name", "curr_date"])
'''
***首先选出计算的股票篮子，然后从行情数据中查询我们需要的部分***
'''
a_stock_normal_dict = db_data_pre_treat.get_normal_stocklist_dict(start_date,
end_date,
year=0,
month=3)
total_stock_list = xyk_common_data_processing.get_all_element_from_dict(
a_stock_normal_dict)
hq_dict = db_interaction.get_daily_data_dict_1_key(start_date, end_date,
"daily_stock_technical",
['liquid_MV', 'close'],
total_stock_list, 0)
stock_suspension_dict = db_data_pre_treat.get_stock_suspension_dict_by_stock(
total_stock_list, start_date, end_date, sus_type=1)
hq_dict_no_suspension = xyk_common_data_processing.get_dict_difference(
hq_dict, stock_suspension_dict, list_order=0)
LN_SHIBOR_dict = {}
for date in SHIBOR_dict.keys():
LN_SHIBOR_dict[date] = math.log(1.0 + SHIBOR_dict[date])
'''
***以下为计算Beta描述量与HSIGMA描述量***
***上市后有After_date个可交易日以上时才进行计算***
***计算二者时使用252个交易日收益率，半衰期均为63***
'''
'Liquidity', 'Earnings', 'Growth', 'Leverage', 'ROR'
]
table_name = "daily_stock_descriptors_pretreated"
a_stock_no_st_dict = db_data_pre_treat.db_data_pre_treat.get_normal_stocklist_dict(
start_date, end_date, year=0, month=6)
total_stock_list = xyk_common_data_processing.get_all_element_from_dict(
a_stock_no_st_dict)
'''
***从描述量数据中查询我们需要的部分，从行情序列中同样进行查询***
'''
#descriptors_1_dict = db_interaction.get_daily_data_dict_1_key(start_date, end_date, "daily_stock_descriptors", descriptor_1_list, total_stock_list, 0, 1)
#descriptors_2_dict = db_interaction.get_daily_data_dict_1_key(start_date, end_date, "daily_stock_descriptors_unified", descriptor_2_list, total_stock_list, 0, 1)
#descriptors_3_dict = db_interaction.get_daily_data_dict_1_key(start_date, end_date, "daily_stock_descriptors_" + Now_Index + "_unique", descriptor_3_list, total_stock_list, 0, 1)
descriptors_4_dict = db_interaction.get_daily_data_dict_1_key(
start_date, end_date, "daily_stock_descriptors", descriptor_4_list,
total_stock_list, 0, 1)
'''
***根据Index篮子确定标准化的参数***
'''
pretreat_list = []
count = 0
for date in descriptors_4_dict.keys():
count += 1
print count
daily_des_value_list = xyk_common_data_processing.exchange_sequence(
descriptors_4_dict[date])
i = 1
while i < len(daily_des_value_list):
'Earnings':[['ETOP', 'EP_Fwd12M', 'CashFlowYield_TTM'], [0.11, 0.68, 0.21]], 'Growth':[['Earnings_STG', 'ROE', 'YOY_Profit'], [0.1, 0.5, 0.4]], \
'Leverage':[['MLEV', 'DTOA', 'BLEV'], [0.38, 0.35, 0.27]]}
Pretreat_Table_Name = 'daily_stock_descriptors_pretreated_' + Now_Index
Factor_Table_Name = 'daily_stock_factors_' + Now_Index
daily_date_list = xyk_common_wind_db_interaction.get_calendar(start_date, end_date, 0)
if Now_Index == "all":
components_dict = db_data_pre_treat.get_normal_stocklist_dict(start_date, end_date, year = 0, month = 6)
else:
where = Now_Index + " = 1"
components_dict = db_interaction.get_data_commonly("daily_index_components", ["stock_id"], ["curr_date"], one_to_one = 0, where = where)
descriptor_data_dict = db_interaction.get_daily_data_dict_1_key(start_date, end_date, Pretreat_Table_Name, Descriptor_List + Keep_Data_List, date_for_key = 1, to_df = 1)
factor_data_dict = {}
for date in daily_date_list:
print date, 'calculating factor values...'
factor_data_dict[date] = descriptor_data_dict[date].loc[:, Keep_Data_List]
for factor in Factor_List:
if len(Factor_Weight_Dict[factor][0]) > 1:
temp_data_df = xyk_common_data_processing.descriptors_aggregate_to_factor(descriptor_data_dict[date].loc[:, Factor_Weight_Dict[factor][0]], Factor_Weight_Dict[factor][1])
else:
temp_data_df = descriptor_data_dict[date].loc[:, Factor_Weight_Dict[factor][0][0]]
temp_data_df.columns = pd.Index([factor])
factor_data_dict[date] = pd.concat([factor_data_dict[date], temp_data_df], axis = 1, join = 'inner')
for date in daily_date_list:
factor_data_dict[date].columns = Keep_Data_List + Factor_List
@author: xiyuk
"""
import math
import numpy as np
import pandas as pd
from datetime import datetime
import datetime as dt
import db_interaction
import xyk_common_data_processing
import xyk_common_wind_db_interaction
import db_data_pre_treat
import csv
start_date = "20180525"
end_date = "20180525"
factor_list = [
'Book_to_Price', 'Earnings', 'Growth', 'Leverage', 'Liquidity', 'Size',
'NL_Size', 'Beta', 'Momentum', 'Residual_Volatility'
]
Now_Index = "zz800"
'''
***从csv中读取考察的组合的权重***
'''
'''
***读取对应的因子数据***
'''
factor_dict = db_interaction.get_daily_data_dict_1_key(
start_date, end_date, "daily_stock_factors_" + Now_Index, factor_list, [],
0, 1, 1)
descriptor_2_list = ['ROE', 'YOY_Profit']
descriptor_3_list = ['LNCAP', 'Long_Momentum', 'Medium_Momentum', 'Short_Momentum', 'DASTD', 'CMRA']
descriptor_4_list = ['Beta', 'HSIGMA']
descriptor_5_list = ['STO_1M', 'STO_3M', 'STO_12M']
Descriptor_List = [descriptor_1_list, descriptor_2_list, descriptor_3_list, descriptor_4_list, descriptor_5_list]
Source_Table_Name_List = ["dp_daily_stock_descriptors_fundamental", "daily_stock_descriptors_fundamental", "daily_stock_descriptors_unified", "daily_stock_descriptors_" + Now_Index + "_unique", "daily_stock_descriptors_unified"]
a_stock_normal_dict = db_data_pre_treat.get_normal_stocklist_dict(start_date, end_date, year = 0, month = 6)
total_stock_list = xyk_common_data_processing.get_all_element_from_dict(a_stock_normal_dict)
i = 0
while i < 6:
print i, 'building dict...'
if i == 0:
raw_data_dict = db_interaction.get_daily_data_dict_1_key(start_date, end_date, Source_Table_Name_List[i], Descriptor_List[i], date_for_key = 1, to_df = 1)
elif i == 5:
raw_data_dict_temp = db_interaction.get_daily_data_dict_1_key(start_date, end_date, "daily_stock_technical", ['liquid_MV', 'close'], date_for_key = 1, to_df = 1)
for date in raw_data_dict.keys():
raw_data_dict[date] = raw_data_dict[date].merge(raw_data_dict_temp[date], how = 'outer', left_index = True, right_index = True)
else:
raw_data_dict_temp = db_interaction.get_daily_data_dict_1_key(start_date, end_date, Source_Table_Name_List[i], Descriptor_List[i], date_for_key = 1, to_df = 1)
for date in raw_data_dict.keys():
raw_data_dict[date] = raw_data_dict[date].merge(raw_data_dict_temp[date], how = 'outer', left_index = True, right_index = True)
i += 1
for date in raw_data_dict.keys():
raw_data_dict[date]['STO_1M'] = np.log(raw_data_dict[date]['STO_1M'])
raw_data_dict[date]['STO_3M'] = np.log(raw_data_dict[date]['STO_3M'])
raw_data_dict[date]['STO_12M'] = np.log(raw_data_dict[date]['STO_12M'])