def TestAllStocksDailyData():
print("dataset: %s" % wave_kernel.FileNameDailyDataSet())
if not os.path.exists(wave_kernel.FileNameDailyDataSet()):
result_df = pd.DataFrame()
increase_sum = 0.0
holding_days_sum = 0
trade_count_sum = 0.0
trade_count_profitable_sum = 0.0
code_list = tushare_data.StockCodes()
pp_merge_data = pp_daily_update.GetPreprocessedMergeData()
for code_index in range(0, len(code_list)):
stock_code = code_list[code_index]
pp_data = pp_daily_update.GetPreprocessedData(
pp_merge_data, stock_code)
if len(pp_data) == 0:
continue
wave_kernel.AppendWaveData(pp_data)
if pridect_mode:
temp_increase, temp_holding_days, trade_count, trade_count_profitable = \
wave_kernel.TradeTest(pp_data, 0.05, False, True, True, False, False, True)
else:
temp_increase, temp_holding_days, trade_count, trade_count_profitable = \
wave_kernel.TradeTest(pp_data, 0.05, False, False, False, False, True, True)
increase_sum += temp_increase
holding_days_sum += temp_holding_days
trade_count_sum += trade_count
trade_count_profitable_sum += trade_count_profitable
if holding_days_sum > 0:
daily_increase = increase_sum / holding_days_sum
else:
daily_increase = 0.0
if trade_count_sum > 0:
avg_increase = increase_sum / trade_count_sum
profitable_ratio = trade_count_profitable_sum / trade_count_sum
else:
avg_increase = 0.0
profitable_ratio = 0.0
if not pridect_mode:
print("%-4d : %s 100%%, %-8.2f, %-8.2f, %-8.2f, %-8.2f, %u/%u:%.2f" % (
code_index, \
stock_code, \
temp_increase, \
increase_sum, \
avg_increase, \
daily_increase, \
trade_count_profitable_sum, \
trade_count_sum, \
profitable_ratio))
# row = {'ts_code':stock_code, 'increase':temp_increase}
# result_df = result_df.append(row, ignore_index=True)
# result_df.to_csv('wave_test_result.csv')
wave_kernel.SaveDailyDataSet()
dataset = wave_dataset.GetDailyDataSet()
wave_test_regression.RegressionTest(dataset)
def TestAllStocks():
result_df = pd.DataFrame()
increase_sum = 0.0
holding_days_sum = 0
trade_count_sum = 0.0
trade_count_profitable_sum = 0.0
code_list = tushare_data.StockCodes()
for code_index in range(0, len(code_list)):
stock_code = code_list[code_index]
temp_increase, temp_holding_days, trade_count, trade_count_profitable = \
TestAStockLowLevel(stock_code, False, False, False, False)
increase_sum += temp_increase
holding_days_sum += temp_holding_days
trade_count_sum += trade_count
trade_count_profitable_sum += trade_count_profitable
if holding_days_sum > 0:
daily_increase = increase_sum / holding_days_sum
else:
daily_increase = 0.0
if trade_count_sum > 0:
avg_increase = increase_sum / trade_count_sum
else:
avg_increase = 0.0
if trade_count_sum > 0:
profitable_ratio = trade_count_profitable_sum / trade_count_sum
else:
profitable_ratio = 0.0
print("%-4d : %s 100%%, %-8.2f, %-8.2f, %-8.2f, %-8.2f, %u/%u:%.2f" % (
code_index, \
stock_code, \
temp_increase, \
increase_sum, \
avg_increase, \
daily_increase, \
trade_count_profitable_sum, \
trade_count_sum, \
profitable_ratio))
row = {'ts_code': stock_code, 'increase': temp_increase}
result_df = result_df.append(row, ignore_index=True)
# print("%-4d : %s 100%%, %-8.2f, %-8.2f, %-8.2f, %-8.2f, %u/%u:%.2f" % (
# code_index, \
# stock_code, \
# temp_increase, \
# increase_sum, \
# avg_increase, \
# daily_increase, \
# trade_count_profitable_sum, \
# trade_count_sum, \
# profitable_ratio))
result_df.to_csv('wave_test_result.csv')
# wave_kernel.SaveTrainData()
wave_kernel.SaveDataSet()
def TestAllStocks():
increase_sum = 0.0
holding_days_sum = 0
code_list = tushare_data.StockCodes()
for code_index in range(0, len(code_list)):
stock_code = code_list[code_index]
temp_increase, temp_holding_days = TestAStockLowLevel(
stock_code, False)
increase_sum += temp_increase
holding_days_sum += temp_holding_days
if holding_days_sum > 0:
daily_increase = increase_sum / holding_days_sum
else:
daily_increase = 0.0
print("%-4d : %s 100%%, %.2f, %.2f, %.2f" %
(code_index, stock_code, temp_increase, increase_sum,
daily_increase))
def CreateDataSetMerge(dataset_file_name):
start_date = dataset_start_date
end_date = tushare_data.train_test_date
date_list = tushare_data.TradeDateListRange(start_date, end_date).tolist()
code_list = tushare_data.StockCodes(dataset_stock_sample_step)
dataset = np.zeros((len(date_list), len(code_list),
feature.feature_size + feature.acture_unit_size))
code_index_map = ListToIndexMap(code_list)
for code_index in range(0, len(code_list)):
stock_code = code_list[code_index]
dataset_split_file_name = FileNameDataSetSplit(stock_code)
if not os.path.exists(dataset_split_file_name):
continue
split_data = np.load(dataset_split_file_name)
dateset_index2 = code_index_map[stock_code]
for iloop in range(len(date_list)):
dataset[iloop][dateset_index2] = split_data[iloop][0]
print("dataset: {}".format(dataset.shape))
print("file_name: %s" % dataset_file_name)
np.save(dataset_file_name, dataset)
def TestAllStocks():
result_df = pd.DataFrame()
amount_sum = 0.0
cost_sum = 0.0
code_list = tushare_data.StockCodes()
for code_index in range(0, len(code_list)):
stock_code = code_list[code_index]
temp_amount, temp_cost = TestAStock(stock_code, False)
amount_sum += temp_amount
cost_sum += temp_cost
avg_increase = (amount_sum - cost_sum) / cost_sum * 100.0
print("%-4d : %s 100%%, %-8.2f, %-8.2f, %-8.2f" % (
code_index, \
stock_code, \
amount_sum, \
cost_sum, \
avg_increase))
temp_increase = (temp_amount - temp_cost) / temp_cost * 100.0
row = {'ts_code': stock_code, 'increase': temp_increase}
result_df = result_df.append(row, ignore_index=True)
result_df.to_csv('fix_trade_test_result.csv')
def CreateDataSetSplit():
start_date = dataset_start_date
end_date = tushare_data.train_test_date
date_list = tushare_data.TradeDateListRange(start_date, end_date).tolist()
code_list = tushare_data.StockCodes(dataset_stock_sample_step)
date_index_map = ListToIndexMap(date_list, True)
data_unit_date_index = ACTURE_DATA_INDEX_DATE()
valid_data_unit_num = 0
for code_index in range(0, len(code_list)):
stock_code = code_list[code_index]
dataset_split_file_name = FileNameDataSetSplit(stock_code)
if not os.path.exists(dataset_split_file_name):
stock_pp_file_name = tushare_data.FileNameStockPreprocessedData(
stock_code)
if os.path.exists(stock_pp_file_name):
pp_data = pd.read_csv(stock_pp_file_name)
else:
pp_data = []
if len(pp_data) == 0:
continue
dataset = np.zeros(
(len(date_list), 1,
feature.feature_size + feature.acture_unit_size))
for day_loop in range(0, len(pp_data)):
data_unit = feature.GetDataUnit1Day(pp_data, day_loop)
if len(data_unit) == 0:
continue
temp_date = int(data_unit[data_unit_date_index])
if temp_date < start_date or temp_date > end_date:
continue
dateset_index1 = date_index_map[pp_data['trade_date']
[day_loop]]
dataset[dateset_index1][0] = data_unit
split_data_date = dataset[:, :, ACTURE_DATA_INDEX_DATE()]
np.save(dataset_split_file_name, dataset)
print("%-4d : %s 100%%" % (code_index, stock_code))
import breakup_kernel
import random
import matplotlib.pyplot as plt
import matplotlib as mpl
from datetime import datetime
import matplotlib.dates as mdate
import preprocess
reload(sys)
sys.setdefaultencoding('utf-8')
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
# update_date = tushare_data.CurrentDate()
# update_date = '20190604'
update_date = '20191224'
code_list = tushare_data.StockCodes()
def CreatePPMergeDataOriginal():
merge_file_name = tushare_data.FileNameMergePPDataOriginal()
print(merge_file_name)
if not os.path.exists(merge_file_name):
merge_pp_data = pd.DataFrame()
for code_index in range(0, len(code_list)):
stock_code = code_list[code_index]
file_name = tushare_data.FileNameStockPreprocessedData(stock_code)
if os.path.exists(file_name):
stock_pp_data = pd.read_csv(file_name)
if len(stock_pp_data) >= 200:
stock_pp_data = stock_pp_data[:400].copy()
merge_pp_data = merge_pp_data.append(stock_pp_data)
def UpdateFixDataSet(is_daily_data,
save_unfinished_record,
dataset_merge=True):
if dataset_merge:
if is_daily_data:
dataset_file_name = FileNameFixDataSetDaily()
pp_merge_data = pp_daily_update.GetPreprocessedMergeData()
else:
dataset_file_name = FileNameFixDataSet()
if os.path.exists(dataset_file_name):
print('dataset already exist: %s' % dataset_file_name)
return
else:
path_name = FileNameFixDataSetPath()
if not os.path.exists(path_name):
os.makedirs(path_name)
code_list = tushare_data.StockCodes()
init_flag = True
for code_index in range(0, len(code_list)):
stock_code = code_list[code_index]
if not dataset_merge:
stock_dataset_file_name = FileNameFixDataSetStock(stock_code)
if os.path.exists(stock_dataset_file_name):
print("%-4d : %s 100%%" % (code_index, stock_code))
continue
if is_daily_data:
pp_data = pp_daily_update.GetPreprocessedData(
pp_merge_data, stock_code)
else:
stock_pp_file_name = tushare_data.FileNameStockPreprocessedData(
stock_code)
if os.path.exists(stock_pp_file_name):
pp_data = pd.read_csv(stock_pp_file_name)
else:
pp_data = []
if len(pp_data) > 0:
pp_data = pp_data[pp_data['trade_date'] >= int(
dataset_start_date)].copy().reset_index(drop=True)
data_list = []
if save_unfinished_record:
valid_data_num = len(pp_data) - feature.feature_days
start_index = 0
else:
valid_data_num = len(
pp_data) - feature.feature_days - feature.label_days
start_index = feature.label_days
if valid_data_num > 0:
for day_loop in range(start_index,
start_index + valid_data_num):
data_unit = feature.GetDataUnit(pp_data, day_loop)
if len(data_unit) > 0:
data_list.append(data_unit)
temp_np_data = np.array(data_list)
if dataset_merge:
if init_flag:
data_set = temp_np_data
init_flag = False
else:
data_set = np.vstack((data_set, temp_np_data))
else:
np.save(stock_dataset_file_name, temp_np_data)
print("%-4d : %s 100%%" % (code_index, stock_code))
# print("train_data: {}".format(train_data.shape))
# print(train_data)
# if (code_index > 0) and ((code_index % 100) == 0):
# print("dataset: {}".format(data_set.shape))
# np.save(dataset_file_name, data_set)
if dataset_merge:
print("dataset: {}".format(data_set.shape))
np.save(dataset_file_name, data_set)