def check_factor(self, name_list, file_name, check_path=None):
if check_path is None:
load_path = os.path.join('/mnt/mfs/dat_whs/data/new_factor_data/' +
self.sector_name)
else:
load_path = check_path
exist_factor = set([x[:-4] for x in os.listdir(load_path)])
use_factor = set(name_list)
a = use_factor - exist_factor
if len(a) != 0:
print('factor not enough!')
send_email.send_email(f'{file_name} factor not enough!',
['*****@*****.**'], [],
'Factor Test Warning!')
def load_raw_data(root_path, raw_data_path, xnms, xinx, if_replace,
target_date):
raw_data_list = []
for target_path in raw_data_path:
tmp_data = bt.AZ_Load_csv(
os.path.join('/media/hdd1/DAT_EQT',
target_path)).reindex(columns=xnms, index=xinx)
if tmp_data.index[-1] != target_date:
send_email.send_email(
target_path + ' Data Error!', ['*****@*****.**'], [],
'[{}]'.format(target_date.strftime('%Y%m%d')))
if if_replace:
tmp_data = tmp_data.replace(0, np.nan)
raw_data_list += [tmp_data]
return raw_data_list
def corr_test_fun(self, pnl_df, alpha_name):
sum_pnl_df = self.get_corr_matrix(cut_date=None)
sum_pnl_df_c = pd.concat([sum_pnl_df, pnl_df], axis=1)
corr_self = sum_pnl_df_c.corr()[[alpha_name]]
other_corr = self.get_all_pnl_corr(pnl_df, alpha_name)
print(other_corr)
self_corr = corr_self[corr_self > 0.6].dropna(axis=0)
print(corr_self.sort_values(by=alpha_name)[-5:])
if len(self_corr) >= 2 or len(other_corr) >= 2:
print('FAIL!')
send_email.send_email('FAIL!\n' + self_corr.to_html(),
['*****@*****.**'], [],
'[RESULT DEAL]' + alpha_name)
else:
print('SUCCESS!')
send_email.send_email('SUCCESS!\n' + self_corr.to_html(),
['*****@*****.**'], [],
'[RESULT DEAL]' + alpha_name)
print('______________________________________')
return 0
def create_data_fun(mode, info_path, sector_df, xnms, xinx, target_date):
info = pd.read_pickle(info_path)
root_path = pt._BinFiles(mode)
args = info['args']
fun_list = info['fun'].split('.')
raw_data_path = list(
map(
lambda x: str(x)[17:]
if str(x).startswith('/mnt/mfs/DAT_EQT') else x,
info['raw_data_path']))
if_replace = info['if_replace']
raw_data_list = load_raw_data(root_path, raw_data_path, xnms, xinx,
if_replace, target_date)
target_fun = find_fun(fun_list)
target_df = target_fun(*raw_data_list, sector_df, *args)
if (target_df.iloc[-1] != 0).sum() == 0:
send_email.send_email(info_path, ['*****@*****.**'], [],
'Data Update Warning')
return target_df
def load_intra_factor(self, file_name):
# load_path = '/mnt/mfs/temp/GTJA_intraday_factor'
load_path = '/media/hdd1/DAT_EQT/INTRADAY/others'
raw_df = bt.AZ_Load_csv(f'{load_path}/{file_name[4:].lower()}.csv')
a = list(set(raw_df.iloc[-1, :100].dropna().values))
if self.xinx[-1] not in raw_df.index:
send_email.send_email(file_name + self.sector_name,
['*****@*****.**'], [], '[LOADDATA]error')
raw_df = raw_df.reindex(index=self.xinx, columns=self.xnms)
tmp_df = raw_df.reindex(index=self.xinx, columns=self.xnms)
if len(a) > 5:
target_df = self.row_extre(tmp_df, self.sector_df, 0.3)
else:
target_df = tmp_df
pass
if self.if_only_long:
target_df = target_df[target_df > 0]
return target_df
