def prepare_train_test_data(self, data_feature, LabelColumnName):
firstloop = 1
for ticker, data in data_feature.items():
X, y = preprocessing_data(self.paras,
data[0],
LabelColumnName,
one_hot_label_proc=False,
array_format=False)
X_train_temp, X_test_temp, y_train_temp, y_test_temp = train_test_split(
X, y, test_size=0.2)
# print('Train shape X:', X_train_temp.shape, ',y:', y_train_temp.shape)
# print('Test shape X:', X_test_temp.shape, ',y:', y_test_temp.shape)
if firstloop == 1:
firstloop = 0
X_train = X_train_temp
X_test = X_test_temp
y_train = y_train_temp
y_test = y_test_temp
else:
X_train.append(X_train_temp, ignore_index=True)
X_test.append(X_test_temp, ignore_index=True)
y_train = np.append(y_train, y_train_temp, 0)
y_test = np.append(y_test, y_test_temp, 0)
# print('Train shape X:', X_train.shape, ',y:', y_train.shape)
# print('Test shape X:', X_test.shape, ',y:', y_test.shape)
return X_train, y_train, X_test, y_test
def prepare_train_test_data(self, data_feature, LabelColumnName):
firstloop = 1
for ticker, data in data_feature.items():
#print(ticker, "n_feature", self.paras.n_features, len(data[0]))
X, y = preprocessing_data(self.paras,
data[0],
LabelColumnName,
one_hot_label_proc=True)
X, y = reshape_input(self.paras.n_features, X, y)
X_train_temp, X_test_temp, y_train_temp, y_test_temp = train_test_split(
X, y, test_size=0.2)
# print('Train shape X:', X_train_temp.shape, ',y:', y_train_temp.shape)
# print('Test shape X:', X_test_temp.shape, ',y:', y_test_temp.shape)
if firstloop == 1:
firstloop = 0
X_train = X_train_temp
X_test = X_test_temp
y_train = y_train_temp
y_test = y_test_temp
else:
X_train = np.append(X_train, X_train_temp, 0)
X_test = np.append(X_test, X_test_temp, 0)
y_train = np.append(y_train, y_train_temp, 0)
y_test = np.append(y_test, y_test_temp, 0)
# print('Train shape X:', X_train.shape, ',y:', y_train.shape)
# print('Test shape X:', X_test.shape, ',y:', y_test.shape)
return X_train, y_train, X_test, y_test
def prepare_train_test_data(self, data_feature, LabelColumnName):
firstloop = 1
for ticker, data in data_feature.items():
X, y = preprocessing_data(self.paras, data[0], LabelColumnName, one_hot_label_proc=False)
X_train_temp, X_test_temp, y_train_temp, y_test_temp = train_test_split(X, y, test_size=0.2)
if firstloop == 1:
firstloop = 0
X_train = X_train_temp
X_test = X_test_temp
y_train = y_train_temp
y_test = y_test_temp
else:
X_train = np.append(X_train, X_train_temp, 0)
X_test = np.append(X_test, X_test_temp, 0)
y_train = np.append(y_train, y_train_temp, 0)
y_test = np.append(y_test, y_test_temp, 0)
return X_train, y_train, X_test, y_test
def predict_data(self, model, data_feature, window, LabelColumnName):
if model == None: model = self.load_training_model(window)
if model == None:
print('predict failed, model not exist')
return
filename = open("./predict_out.txt", 'w')
for ticker in self.paras.predict_tickers:
try:
data = data_feature[ticker]
except:
# print('stock not preparee', ticker)
continue
X_train, y_train = preprocessing_data(self.paras,
data[0],
LabelColumnName,
one_hot_label_proc=True)
X_valid, y_valid = preprocessing_data(self.paras,
data[1],
LabelColumnName,
one_hot_label_proc=True)
X_lately, y_lately = preprocessing_data(self.paras,
data[2],
LabelColumnName,
one_hot_label_proc=False)
X_train, y_train = reshape_input(self.paras.n_features, X_train,
y_train)
X_valid, y_valid = reshape_input(self.paras.n_features, X_valid,
y_valid)
X_lately, y_lately = reshape_input(self.paras.n_features, X_lately,
y_lately)
possibility_columns = [
str(window) + '_' + str(idx)
for idx in range(self.paras.n_out_class)
]
# print('\n ---------- ', ticker, ' ---------- \n')
# print(' ############## validation on train data ############## ')
mse_known_train, predictions_train = self.predict(
model, X_train, y_train)
data[3].loc[data[0].index, 'label'] = np.argmax(
y_train, axis=1) #- int(self.paras.n_out_class/2)
data[3].loc[data[0].index, 'pred'] = np.argmax(
predictions_train, axis=1) #- int(self.paras.n_out_class/2)
s = pd.DataFrame(predictions_train,
index=data[0].index,
columns=possibility_columns)
# print(' ############## validation on valid data ############## ')
mse_known_lately, predictions_valid = self.predict(
model, X_valid, y_valid)
data[3].loc[data[1].index, 'label'] = np.argmax(
y_valid, axis=1) #- int(self.paras.n_out_class/2)
data[3].loc[data[1].index, 'pred'] = np.argmax(
predictions_valid, axis=1) #- int(self.paras.n_out_class/2)
s = s.append(
pd.DataFrame(predictions_valid,
index=data[1].index,
columns=possibility_columns))
# print(' ############## validation on lately data ############## ')
mse_lately, predictions_lately = self.predict(
model, X_lately, y_lately)
data[3].loc[data[2].index,
'label'] = np.nan #np.argmax(actual_lately, axis=1)
data[3].loc[data[2].index, 'pred'] = np.argmax(
predictions_lately, axis=1) #- int(self.paras.n_out_class/2)
s = s.append(
pd.DataFrame(predictions_lately,
index=data[2].index,
columns=possibility_columns))
data[3] = pd.merge(data[3],
s,
how='outer',
left_index=True,
right_index=True)
if data[3]['pred'][-1] == 3:
continue
actual_count = []
predict_count = []
for i in range(self.paras.n_out_class):
actual_count.append(len(data[3][data[3]['label'] == i]))
predict_count.append(
len(data[3][(data[3]['label'] == i)
& (data[3]['label'] == data[3]['pred'])]))
valid_actual_count = []
valid_predict_count = []
data.append(data[3][-self.paras.valid_len:])
for i in range(self.paras.n_out_class):
valid_actual_count.append(len(data[4][data[4]['label'] == i]))
valid_predict_count.append(
len(data[4][(data[4]['label'] == i)
& (data[4]['label'] == data[4]['pred'])]))
# print('classification counter:\n', actual_count)
# print('classification possibility:\n', 100*np.array(actual_count)/np.sum(actual_count))
# print('classification train predict:\n', 100*np.array(predict_count)/np.array(actual_count))
# print('classification valid predict:\n', 100*np.array(valid_predict_count)/np.array(valid_actual_count))
# timePeriod = [22*24, 22*12, 22*6, 22*3, 22*2, 22]
# pred_profit = data[3]["pred_profit"]
# pred_profit_len = len(pred_profit)
# centers_oris = []
# index_oris = []
# for time in timePeriod:
# if pred_profit_len < time: continue
# out_labels, counters, centers_ori = kmeans_claasification(pred_profit[pred_profit_len - time : pred_profit_len], self.paras.n_out_class)
# centers_oris.append(centers_ori)
# index_oris.append("Days: " + str(time))
# df_ori = pd.DataFrame(centers_oris, index=index_oris, columns=[str(idx) for idx in range(self.paras.n_out_class)])
# print('\nclassification centers:\n', df_ori)
data[3]['label'] = data[3]['label'] - int(
self.paras.n_out_class / 2)
data[3]['pred'] = data[3]['pred'] - int(self.paras.n_out_class / 2)
# rewrite data frame and save / update
data[3] = self.save_data_frame_mse(
ticker,
data[3],
window,
possibility_columns,
mses=[mse_known_train, mse_known_lately])
self.df = data[3]
pd.set_option('display.max_rows', None)
print('\n ---------- ', ticker, ' ---------- \n', file=filename)
print(data[3][-(self.paras.pred_len + self.paras.valid_len):],
file=filename,
flush=True)
def predict_data(self, model, data_feature, LabelColumnName, index):
if model == None:
model = self.load_training_model(self.paras.window_len[index])
if model == None:
print('predict failed, model not exist')
return
for ticker in self.paras.predict_tickers:
try:
data = data_feature[ticker]
except:
print('stock not prepare', ticker)
continue
X_train, y_train = preprocessing_data(self.paras,
data[0],
LabelColumnName,
one_hot_label_proc=False)
X_valid, y_valid = preprocessing_data(self.paras,
data[1],
LabelColumnName,
one_hot_label_proc=False)
X_lately, y_lately = preprocessing_data(self.paras,
data[2],
LabelColumnName,
one_hot_label_proc=False)
possibility_columns = [
str(self.paras.window_len[index]) + '_' + str(idx)
for idx in range(self.paras.n_out_class)
]
print('\n ---------- ', ticker, ' ---------- \n')
print(' ############## validation on train data ############## ')
predictions_train = self.predict(model, X_train, y_train)
data[3].loc[data[0].index,
'actual'] = y #- int(self.paras.n_out_class/2)
data[3].loc[data[0].index, 'pred'] = np.argmax(
predictions_train, axis=1) #- int(self.paras.n_out_class/2)
s = pd.DataFrame(predictions_train,
index=data[0].index,
columns=possibility_columns)
print(' ############## validation on valid data ############## ')
predictions_valid = self.predict(model, X_valid, y_valid)
data[3].loc[data[1].index,
'actual'] = y_valid #- int(self.paras.n_out_class/2)
data[3].loc[data[1].index, 'pred'] = np.argmax(
predictions_valid, axis=1) #- int(self.paras.n_out_class/2)
s = s.append(
pd.DataFrame(predictions_valid,
index=data[1].index,
columns=possibility_columns))
print(' ############## validation on lately data ############## ')
predictions_lately = self.predict(model, X_lately, y_lately)
data[3].loc[data[2].index,
'actual'] = np.nan #np.argmax(actual_lately, axis=1)
data[3].loc[data[2].index, 'pred'] = np.argmax(
predictions_lately, axis=1) #- int(self.paras.n_out_class/2)
s = s.append(
pd.DataFrame(predictions_lately,
index=data[2].index,
columns=possibility_columns))
data[3] = pd.merge(data[3],
s,
how='outer',
left_index=True,
right_index=True)
actual_count = []
predict_count = []
for i in range(self.paras.n_out_class):
actual_count.append(len(data[3][data[3]['actual'] == i]))
predict_count.append(
len(data[3][(data[3]['actual'] == i)
& (data[3]['actual'] == data[3]['pred'])]))
valid_actual_count = []
valid_predict_count = []
data.append(data[3][-self.paras.valid_len:])
for i in range(self.paras.n_out_class):
valid_actual_count.append(len(data[4][data[4]['actual'] == i]))
valid_predict_count.append(
len(data[4][(data[4]['actual'] == i)
& (data[4]['actual'] == data[4]['pred'])]))
print('classification counter:\n', actual_count)
print('classification possibility:\n',
100 * np.array(actual_count) / np.sum(actual_count))
print('classification train predict:\n',
100 * np.array(predict_count) / np.array(actual_count))
print(
'classification valid predict:\n', 100 *
np.array(valid_predict_count) / np.array(valid_actual_count))
#print('\nclassification centers:\n', np.round(np.sort(data[5], axis=1), decimals=3))
data[3]['actual'] = data[3]['actual'] - int(
self.paras.n_out_class / 2)
data[3]['pred'] = data[3]['pred'] - int(self.paras.n_out_class / 2)
# rewrite data frame and save / update
data[3] = self.save_data_frame_mse(ticker, data[3],
self.paras.window_len[index],
possibility_columns)
self.df.append(data[3])
pd.set_option('display.max_rows', None)
print('\n -------------------- \n')
print(data[3][-(self.paras.pred_len + self.paras.valid_len):])
def predict_data(self, model, data_feature, LabelColumnName, index):
if model == None:
model = self.load_training_model(self.paras.window_len[index])
if model == None:
print('predict failed, model not exist')
return
for ticker in self.paras.predict_tickers:
try:
data = data_feature[ticker]
except:
print('stock not preparee', ticker)
continue
X_train, y_train = preprocessing_data(self.paras,
data[0],
LabelColumnName,
one_hot_label_proc=False)
X_valid, y_valid = preprocessing_data(self.paras,
data[1],
LabelColumnName,
one_hot_label_proc=False)
X_lately, y_lately = preprocessing_data(self.paras,
data[2],
LabelColumnName,
one_hot_label_proc=False)
possibility_columns = [
str(self.paras.window_len[index]) + '_' + str(idx)
for idx in range(self.paras.n_out_class)
]
# 奇怪的是 n_out_class 只有 0_0 这是怎么回事呢？？？？？
possibility_columns = possibility_columns[:1]
print('\n ---------- ', ticker, ' ---------- \n')
# print(' ############## validation on train data ############## ')
predictions_train = self.predict(model, X_train, y_train)
data[3].loc[
data[0].index,
'label'] = y_train #np.argmax(y, axis=1) #- int(self.paras.n_out_class/2)
data[3].loc[
data[0].index,
'pred'] = predictions_train # np.argmax(predictions_train, axis=len(predictions_train.shape)-1) #- int(self.paras.n_out_class/2)
s = pd.DataFrame(predictions_train,
index=data[0].index,
columns=possibility_columns)
# print(' ############## validation on valid data ############## ')
predictions_valid = self.predict(model, X_valid, y_valid)
data[3].loc[
data[1].index,
'label'] = y_valid #np.argmax(y_valid, axis=1) #- int(self.paras.n_out_class/2)
data[3].loc[
data[1].index,
'pred'] = predictions_valid #np.argmax(predictions_valid, axis=len(predictions_train.shape)-1) #- int(self.paras.n_out_class/2)
s = s.append(
pd.DataFrame(predictions_valid,
index=data[1].index,
columns=possibility_columns))
# print(' ############## validation on lately data ############## ')
predictions_lately = self.predict(model, X_lately, y_lately)
data[3].loc[data[2].index,
'label'] = np.nan #np.argmax(actual_lately, axis=1)
data[3].loc[data[2].index, 'pred'] = np.argmax(
predictions_lately, axis=len(predictions_train.shape) -
1) #- int(self.paras.n_out_class/2)
s = s.append(
pd.DataFrame(predictions_lately,
index=data[2].index,
columns=possibility_columns))
data[3] = pd.merge(data[3],
s,
how='outer',
left_index=True,
right_index=True)
actual_count = []
predict_count = []
for i in range(self.paras.n_out_class):
actual_count.append(len(data[3][data[3]['label'] == i]))
predict_count.append(
len(data[3][(data[3]['label'] == i)
& (data[3]['label'] == data[3]['pred'])]))
valid_actual_count = []
valid_predict_count = []
data.append(data[3][-self.paras.valid_len:])
for i in range(self.paras.n_out_class):
valid_actual_count.append(len(data[4][data[4]['label'] == i]))
valid_predict_count.append(
len(data[4][(data[4]['label'] == i)
& (data[4]['label'] == data[4]['pred'])]))
# print('\nclassification counter:\n', actual_count)
# print('\nclassification possibility:\n', 100*np.array(actual_count)/np.sum(actual_count))
# print('\nclassification train predict:\n', 100*np.array(predict_count)/np.array(actual_count))
# print('\nclassification valid predict:\n', 100*np.array(valid_predict_count)/np.array(valid_actual_count))
timePeriod = [22 * 24, 22 * 12, 22 * 6, 22 * 3, 22 * 2, 22]
pred_profit = data[3]["pred_profit"]
pred_profit_len = len(pred_profit)
centers_oris = []
index_oris = []
for time in timePeriod:
if pred_profit_len < time: continue
out_labels, counters, centers_ori = kmeans_claasification(
pred_profit[pred_profit_len - time:pred_profit_len],
self.paras.n_out_class)
centers_oris.append(np.sort(centers_ori))
index_oris.append(time)
df_ori = pd.DataFrame(centers_oris, index=index_oris)
df_ori.index.name = 'Days'
# print('\nclassification centers:\n', df_ori)
# rewrite data frame and save / update
data[3] = self.save_data_frame_mse(ticker, data[3],
self.paras.window_len[index],
possibility_columns)
self.df = data[3]
pd.set_option('display.max_rows', None)
# print('\n -------------------- \n')
# data[3]['label'] = data[3]['label'] - int(self.paras.n_out_class/2)
# data[3]['pred'] = data[3]['pred'] - int(self.paras.n_out_class/2)
# print(data[3][-(self.paras.pred_len + self.paras.valid_len):])
print(data[3][-self.paras.pred_len:])
def predict_data(self, model, data_feature, window, LabelColumnName):
if model == None: model = self.load_training_model(window)
if model == None:
print('predict failed, model not exist')
return
filename = open("./predict_out.txt", 'w')
for ticker in self.paras.predict_tickers:
try:
data = data_feature[ticker]
except:
#print('stock not prepare:', ticker)
continue
# print(ticker, len(data[0]), len(data[1]), len(data[2]), len(data[3]))
# print(data[3])
# print("$"*40)
# print(data[0])
X_train, y_train = preprocessing_data(self.paras,
data[0],
LabelColumnName,
one_hot_label_proc=False)
X_valid, y_valid = preprocessing_data(self.paras,
data[1],
LabelColumnName,
one_hot_label_proc=False)
X_lately, y_lately = preprocessing_data(self.paras,
data[2],
LabelColumnName,
one_hot_label_proc=False)
print('\n ---------- ', ticker, ' ---------- \n')
# print('############## validation on train data ##############')
rec_known_train, predictions_train = self.predict(
model, X_train, y_train)
rec_known_valid, _ = self.predict(model, X_valid, y_valid)
data[3].loc[data[0].index,
'label'] = y_train #- int(self.paras.n_out_class/2)
data[3].loc[
data[0].index,
'pred'] = predictions_train #- int(self.paras.n_out_class/2)
mse_known_lately, predictions_valid = self.predict(
model, X_valid, y_valid)
data[3].loc[data[1].index,
'label'] = y_valid #- int(self.paras.n_out_class/2)
data[3].loc[
data[1].index,
'pred'] = predictions_valid #- int(self.paras.n_out_class/2)
mse_lately, predictions_lately = self.predict(
model, X_lately, y_lately)
data[3].loc[data[2].index,
'label'] = np.nan #np.argmax(actual_lately, axis=1)
data[3].loc[
data[2].index,
'pred'] = predictions_lately #- int(self.paras.n_out_class/2)
if predictions_lately[0] == 3:
continue
actual_count = []
predict_count = []
for i in range(self.paras.n_out_class):
actual_count.append(len(data[3][data[3]['label'] == i]))
predict_count.append(
len(data[3][(data[3]['label'] == i)
& (data[3]['label'] == data[3]['pred'])]))
valid_actual_count = []
valid_predict_count = []
data.append(data[3][-self.paras.valid_len:])
for i in range(self.paras.n_out_class):
valid_actual_count.append(len(data[4][data[4]['label'] == i]))
valid_predict_count.append(
len(data[4][(data[4]['label'] == i)
& (data[4]['label'] == data[4]['pred'])]))
data[3]['label'] = data[3]['label'] - int(
self.paras.n_out_class / 2)
data[3]['pred'] = data[3]['pred'] - int(self.paras.n_out_class / 2)
self.df = data[3]
pred_df = data[3]['pred'][-(self.paras.pred_len +
self.paras.valid_len):]
pd.set_option('display.max_rows', None)
if (pred_df == 0).all() == False:
print('\n ---------- ',
ticker,
' ---------- \n',
file=filename)
print(data[3][-(self.paras.pred_len + self.paras.valid_len):],
file=filename,
flush=True)
def predict_data(self, model, data_feature, window, LabelColumnName):
if model == None: model = self.load_training_model(window)
if model == None:
print('predict failed, model not exist')
return
filename = open("./predict_out.txt", 'w')
for ticker in self.paras.predict_tickers:
try:
data = data_feature[ticker]
except:
# print('stock not preparee', ticker)
continue
X_train, y_train = preprocessing_data(self.paras, data[0], LabelColumnName, one_hot_label_proc=True)
X_valid, y_valid = preprocessing_data(self.paras, data[1], LabelColumnName, one_hot_label_proc=True)
X_lately, y_lately = preprocessing_data(self.paras, data[2], LabelColumnName, one_hot_label_proc=False)
X_train, y_train = reshape_input(self.paras.n_features, X_train, y_train)
X_valid, y_valid = reshape_input(self.paras.n_features, X_valid, y_valid)
X_lately, y_lately = reshape_input(self.paras.n_features, X_lately, y_lately)
possibility_columns = [str(window) + '_' + str(idx) for idx in range(self.paras.n_out_class)]
mse_known_train, predictions_train = self.predict(model, X_train, y_train)
data[3].loc[data[0].index, 'label'] = np.argmax(y_train, axis=1) #- int(self.paras.n_out_class/2)
data[3].loc[data[0].index, 'pred'] = np.argmax(predictions_train, axis=1) #- int(self.paras.n_out_class/2)
s = pd.DataFrame(predictions_train, index = data[0].index, columns=possibility_columns)
mse_known_lately, predictions_valid = self.predict(model, X_valid, y_valid)
data[3].loc[data[1].index, 'label'] = np.argmax(y_valid, axis=1) #- int(self.paras.n_out_class/2)
data[3].loc[data[1].index, 'pred'] = np.argmax(predictions_valid, axis=1) #- int(self.paras.n_out_class/2)
s = s.append(pd.DataFrame(predictions_valid, index = data[1].index, columns=possibility_columns))
mse_lately, predictions_lately = self.predict(model, X_lately, y_lately)
data[3].loc[data[2].index, 'label'] = np.nan#np.argmax(actual_lately, axis=1)
data[3].loc[data[2].index, 'pred'] = np.argmax(predictions_lately, axis=1) #- int(self.paras.n_out_class/2)
s = s.append(pd.DataFrame(predictions_lately, index = data[2].index, columns=possibility_columns))
data[3] = pd.merge(data[3], s, how='outer', left_index=True, right_index=True)
if data[3]['pred'][-1] == 3:
continue
actual_count = []
predict_count = []
for i in range(self.paras.n_out_class):
actual_count.append(len(data[3][data[3]['label'] == i]))
predict_count.append(len(data[3][(data[3]['label'] == i) & (data[3]['label'] == data[3]['pred'])]))
valid_actual_count = []
valid_predict_count = []
data.append(data[3][-self.paras.valid_len:])
for i in range(self.paras.n_out_class):
valid_actual_count.append(len(data[4][data[4]['label'] == i]))
valid_predict_count.append(len(data[4][(data[4]['label'] == i) & (data[4]['label'] == data[4]['pred'])]))
data[3]['label'] = data[3]['label'] - int(self.paras.n_out_class/2)
data[3]['pred'] = data[3]['pred'] - int(self.paras.n_out_class/2)
data[3] = self.save_data_frame_mse(ticker, data[3], window, possibility_columns, mses=[mse_known_train, mse_known_lately])
self.df = data[3]
pd.set_option('display.max_rows', None)
print('\n ---------- ', ticker, ' ---------- \n', file = filename)
print(data[3][-(self.paras.pred_len + self.paras.valid_len):], file = filename, flush = True)
