def DownloadStockData(self, ts_code):
if ts_code in self.index_codes:
self.DownloadIndexData(ts_code)
return
start_date = '19000101'
end_date = self.end_date
name_list = self.FileNameStockDownloadData(ts_code)
download_flag = False
while True:
file_name = name_list[0]
if not os.path.exists(file_name):
download_flag = True
df = pro.daily(ts_code = ts_code, start_date = start_date, end_date = end_date)
base_common.MKFileDirs(file_name)
df.to_csv(file_name)
if self.InvalidFileClean(file_name):
continue
file_name = name_list[1]
if not os.path.exists(file_name) and self.use_daily_basic:
download_flag = True
df = pro.daily_basic(ts_code = ts_code, start_date = start_date, end_date = end_date)
base_common.MKFileDirs(file_name)
df.to_csv(file_name)
if self.InvalidFileClean(file_name):
continue
file_name = name_list[2]
if not os.path.exists(file_name) and self.use_money_flow:
download_flag = True
df = pro.moneyflow(ts_code = ts_code, start_date = start_date, end_date = end_date)
base_common.MKFileDirs(file_name)
df.to_csv(file_name)
if self.InvalidFileClean(file_name):
continue
file_name = name_list[3]
if not os.path.exists(file_name) and self.use_adj_factor:
download_flag = True
df = pro.adj_factor(ts_code = ts_code, trade_date='')
col_date = df['trade_date'].copy()
col_date = pd.to_numeric(col_date)
df = df[col_date <= int(end_date)].copy().reset_index(drop=True)
base_common.MKFileDirs(file_name)
df.to_csv(file_name)
if self.InvalidFileClean(file_name):
continue
break
if download_flag:
self.PrintProcess(ts_code)
def TradeTestAll(self, save_data_unit=True, show_trade_record=False):
sum_increase = 0.0
sum_trade_count = 0
sum_holding_days = 0
for iloop in range(len(self.data_source.code_list)):
ts_code = self.data_source.code_list[iloop]
pp_data = self.data_source.LoadStockPPData(ts_code, True)
increase, trade_count, holding_days = self.TradeTest(
pp_data, False, save_data_unit, show_trade_record)
sum_increase += increase
sum_trade_count += trade_count
sum_holding_days += holding_days
base_common.PrintTrade(sum_trade_count, '--', '--', '--', '--', '--',
sum_increase, sum_holding_days)
if sum_trade_count == 0:
avg_increase_stock = 0.0
avg_increase_day = 0.0
avg_increase_trade = 0.0
else:
avg_increase_stock = sum_increase / len(self.data_source.code_list)
avg_increase_day = sum_increase / sum_holding_days
avg_increase_trade = sum_increase / sum_trade_count
print('%-24s: %.4f' % ('avg increase / stock', avg_increase_stock))
print('%-24s: %.4f' % ('avg increase / trade', avg_increase_trade))
print('%-24s: %.4f' % ('avg increase / day', avg_increase_day))
if self.dataset_len > 0:
file_name = self.FileNameDataset()
base_common.MKFileDirs(file_name)
np.save(file_name, self.dataset[:self.dataset_len])
def CreateDSFa3DDataset(self):
dataset_file_name = self.FileNameDSFa3DDataset()
self.dataset = np.zeros(
(len(self.date_list), len(self.code_list), self.feature.unit_size))
# base_common.ListMultiThread(CreateDSFa3DSplitMTFunc, self, 1, self.code_list)
for ts_code in self.code_list:
self.CreateDSFa3DSplitStock(ts_code)
base_common.MKFileDirs(dataset_file_name)
np.save(dataset_file_name, self.dataset)
self.dataset = None
def DownloadTradeDayData(self, trade_date):
name_list = self.FileNameTradeDayDownloadData(trade_date)
download_flag = False
while True:
file_name = name_list[0]
if not os.path.exists(file_name):
download_flag = True
df = pro.daily(trade_date = trade_date)
base_common.MKFileDirs(file_name)
df.to_csv(file_name)
if self.InvalidFileClean(file_name):
continue
file_name = name_list[1]
if not os.path.exists(file_name) and self.use_daily_basic:
download_flag = True
df = pro.daily_basic(trade_date = trade_date)
base_common.MKFileDirs(file_name)
df.to_csv(file_name)
if self.InvalidFileClean(file_name):
continue
file_name = name_list[2]
if not os.path.exists(file_name) and self.use_money_flow:
download_flag = True
df = pro.moneyflow(trade_date = trade_date)
base_common.MKFileDirs(file_name)
df.to_csv(file_name)
if self.InvalidFileClean(file_name):
continue
file_name = name_list[3]
if not os.path.exists(file_name) and self.use_adj_factor:
download_flag = True
df = pro.adj_factor(ts_code='', trade_date=trade_date)
base_common.MKFileDirs(file_name)
df.to_csv(file_name)
if self.InvalidFileClean(file_name):
continue
break
if download_flag:
sys.stdout.write("%s : 100%%\n" % trade_date)
def InitDSWWsUpNumDataset(self):
dataset_file_name = self.FileNameDSWWsUpNumDataset()
if not os.path.exists(dataset_file_name):
dsw_dataset = self.GetDSW3DDataset()
date_num = len(self.data_source.date_list)
dataset = np.zeros((date_num))
for iloop in range(date_num):
dataset[iloop] = np.sum(dsw_dataset[iloop, :, 0] == WS_UP)
base_common.MKFileDirs(dataset_file_name)
np.save(dataset_file_name, dataset)
dataset = np.load(dataset_file_name)
self.dsw_ws_up_num = np.load(dataset_file_name)
def UpdateIndexPPData(self, ts_code):
stock_pp_file_name = self.FileNameStockPPData(ts_code)
if not os.path.exists(stock_pp_file_name):
df = self.LoadDownloadIndexData(ts_code)
pp_data = preprocess.StockDataPreProcess(df, '')
if len(pp_data) > 0:
base_common.MKFileDirs(stock_pp_file_name)
np.save(stock_pp_file_name, pp_data)
else:
print("UpdateStockPPData error: %s" % ts_code)
return
sys.stdout.write("%-4s : %s 100%%\n" % ('--', ts_code))
def TradeDateLowLevel(end_date):
file_name = './data/trade_date/trade_date_%s.npy' % str(end_date)
if os.path.exists(file_name):
date_np = np.load(file_name)
return date_np
else:
df_trade_cal = pro.trade_cal(exchange = 'SSE', start_date = '19800101', end_date = str(end_date))
df_trade_cal = df_trade_cal.sort_index(ascending = False)
df_trade_cal = df_trade_cal[df_trade_cal['is_open'] == 1]
date_np = df_trade_cal['cal_date'].values
base_common.MKFileDirs(file_name)
np.save(file_name, date_np)
return date_np
def UpdateStockPPData(self, ts_code):
if ts_code in self.index_codes:
self.UpdateIndexPPData(ts_code)
return
stock_pp_file_name = self.FileNameStockPPData(ts_code)
if not os.path.exists(stock_pp_file_name):
merge_df = self.LoadDownloadStockData(ts_code)
pp_data = preprocess.StockDataPreProcess(merge_df, self.adj_mode)
if len(pp_data) > 0:
base_common.MKFileDirs(stock_pp_file_name)
np.save(stock_pp_file_name, pp_data)
else:
print("UpdateStockPPData error: %s" % ts_code)
return
self.PrintProcess(ts_code)
def DownloadIndexData(self, ts_code):
start_date = '19000101'
end_date = self.end_date
name_list = self.FileNameStockDownloadData(ts_code)
if not os.path.exists(name_list[0]):
while True:
file_name = name_list[0]
if not os.path.exists(file_name):
df = pro.index_daily(ts_code=ts_code,
start_date=start_date,
end_date=end_date)
base_common.MKFileDirs(file_name)
df.to_csv(file_name)
if self.InvalidFileClean(file_name):
continue
break
sys.stdout.write("%-4s : %s 100%%\n" % ('--', ts_code))
def StockCodesName(release_end_date, industry_filter, code_filter, sample_step, sample_start_offset=0):
file_name = './data/' + 'stock_code' + '.csv'
if os.path.exists(file_name):
load_df = pd.read_csv(file_name)
else:
load_df = pro.stock_basic(exchange = '', list_status = 'L', fields = 'ts_code,symbol,name,area,industry,list_date')
base_common.MKFileDirs(file_name)
load_df.to_csv(file_name)
if release_end_date > 0:
load_df = load_df[load_df['list_date'] <= int(release_end_date)]
if sample_step > 1:
# load_df[::2] # 奇数行
# load_df[1::2] # 偶数行
load_df = load_df[sample_start_offset::sample_step]
load_df = load_df.copy()
load_df = load_df.reset_index(drop=True)
industry_filter_en = False
code_filter_en = False
circ_mv_filter_en = False
if industry_filter != '':
industry_list = industry_filter.split(',')
industry_filter_en = True
if code_filter != '':
code_filter_list = code_filter.split(',')
code_filter_en = True
code_valid_list = []
for iloop in range(0, len(load_df)):
temp_code = load_df['ts_code'][iloop]
temp_code_valid = True
if industry_filter_en:
if not load_df['industry'][iloop] in industry_list:
temp_code_valid = False
if code_filter_en:
if not StockCodeFilter(temp_code, code_filter_list):
temp_code_valid = False
code_valid_list.append(temp_code_valid)
load_df = load_df[code_valid_list]
# print(load_df)
code_list = load_df['ts_code'].values
name_list = load_df['name'].values
return code_list, name_list
def GetDSW3DDataset(self):
dataset_file_name = self.FileNameDSW3DDataset()
if not os.path.exists(dataset_file_name):
dataset = np.zeros((len(self.data_source.date_list),
len(self.data_source.code_list), 1))
for code_index in range(0, len(self.data_source.code_list)):
ts_code = self.data_source.code_list[code_index]
S_index = self.data_source.code_index_map[ts_code]
pp_data = self.data_source.LoadStockPPData(ts_code, True)
data_len = len(pp_data)
if data_len == 0:
continue
self.TradePP(pp_data)
for iloop in range(data_len):
D_index = self.data_source.date_index_map[int(
pp_data[iloop][PPI_trade_date])]
dataset[D_index][S_index][0] = self.wave_data[iloop]
sys.stdout.write("%-4d : %s 100%%\n" % (code_index, ts_code))
base_common.MKFileDirs(dataset_file_name)
np.save(dataset_file_name, dataset)
dataset = np.load(dataset_file_name)
print("dataset: {}".format(dataset.shape))
return dataset