def check_factor(self, name_list, file_name, check_path=None): if check_path is None: load_path = os.path.join('/mnt/mfs/dat_whs/data/new_factor_data/' + self.sector_name) else: load_path = check_path exist_factor = set([x[:-4] for x in os.listdir(load_path)]) use_factor = set(name_list) a = use_factor - exist_factor if len(a) != 0: print('factor not enough!') send_email.send_email(f'{file_name} factor not enough!', ['*****@*****.**'], [], 'Factor Test Warning!')
def load_raw_data(root_path, raw_data_path, xnms, xinx, if_replace, target_date): raw_data_list = [] for target_path in raw_data_path: tmp_data = bt.AZ_Load_csv( os.path.join('/media/hdd1/DAT_EQT', target_path)).reindex(columns=xnms, index=xinx) if tmp_data.index[-1] != target_date: send_email.send_email( target_path + ' Data Error!', ['*****@*****.**'], [], '[{}]'.format(target_date.strftime('%Y%m%d'))) if if_replace: tmp_data = tmp_data.replace(0, np.nan) raw_data_list += [tmp_data] return raw_data_list
def corr_test_fun(self, pnl_df, alpha_name): sum_pnl_df = self.get_corr_matrix(cut_date=None) sum_pnl_df_c = pd.concat([sum_pnl_df, pnl_df], axis=1) corr_self = sum_pnl_df_c.corr()[[alpha_name]] other_corr = self.get_all_pnl_corr(pnl_df, alpha_name) print(other_corr) self_corr = corr_self[corr_self > 0.6].dropna(axis=0) print(corr_self.sort_values(by=alpha_name)[-5:]) if len(self_corr) >= 2 or len(other_corr) >= 2: print('FAIL!') send_email.send_email('FAIL!\n' + self_corr.to_html(), ['*****@*****.**'], [], '[RESULT DEAL]' + alpha_name) else: print('SUCCESS!') send_email.send_email('SUCCESS!\n' + self_corr.to_html(), ['*****@*****.**'], [], '[RESULT DEAL]' + alpha_name) print('______________________________________') return 0
def create_data_fun(mode, info_path, sector_df, xnms, xinx, target_date): info = pd.read_pickle(info_path) root_path = pt._BinFiles(mode) args = info['args'] fun_list = info['fun'].split('.') raw_data_path = list( map( lambda x: str(x)[17:] if str(x).startswith('/mnt/mfs/DAT_EQT') else x, info['raw_data_path'])) if_replace = info['if_replace'] raw_data_list = load_raw_data(root_path, raw_data_path, xnms, xinx, if_replace, target_date) target_fun = find_fun(fun_list) target_df = target_fun(*raw_data_list, sector_df, *args) if (target_df.iloc[-1] != 0).sum() == 0: send_email.send_email(info_path, ['*****@*****.**'], [], 'Data Update Warning') return target_df
def load_intra_factor(self, file_name): # load_path = '/mnt/mfs/temp/GTJA_intraday_factor' load_path = '/media/hdd1/DAT_EQT/INTRADAY/others' raw_df = bt.AZ_Load_csv(f'{load_path}/{file_name[4:].lower()}.csv') a = list(set(raw_df.iloc[-1, :100].dropna().values)) if self.xinx[-1] not in raw_df.index: send_email.send_email(file_name + self.sector_name, ['*****@*****.**'], [], '[LOADDATA]error') raw_df = raw_df.reindex(index=self.xinx, columns=self.xnms) tmp_df = raw_df.reindex(index=self.xinx, columns=self.xnms) if len(a) > 5: target_df = self.row_extre(tmp_df, self.sector_df, 0.3) else: target_df = tmp_df pass if self.if_only_long: target_df = target_df[target_df > 0] return target_df