def download_from_inxfile(filepath):
type = ""
if "=" in filepath:
type, filename = filepath.split("=")
else:
if "inx" in filepath:
type = "inx"
if "stk" in filepath:
type = "stk"
filename = filepath
if len(filename) <= 0:
ta.g.log.error("download params error!")
return
if not my_tools.path_exists(filename):
filename = ta.g.data_path + filename
if not my_tools.path_exists(filename):
ta.g.log.error(filename + " is not exists")
return
down_obj = download()
if type == "inx":
down_obj.download_inx(ta.g.config.inx_path, filename)
if type == "stk":
down_obj.downlaod_stk(ta.g.stk_path, filename)
def downlaod_stk(self, downpath = ta.g.data_path, filename = "stk_code.csv"):
if my_tools.path_exists(filename) == False:
ta.g.log.error("stk file is not exists and exit")
return False
if my_tools.path_exists(downpath) == False:
my_tools.mkdir(downpath)
xtyp = 'D' # xtyp = '5'
zddown.down_stk_all(downpath, filename, xtyp)
def predict(params):
type, code, datafile, modfile, lstfile = prepared(params)
if not my_tools.path_exists(modfile):
modeling(params)
my_modelex.predict(code, datafile, modfile)
def eva(self, mod_filename):
if my_tools.path_exists(mod_filename):
model = self.setmod(mod_filename)
eva_obj = eva.evaluation(self.do)
x_test = my_dm.util.get_features(
self.do.df,
ta.g.config['data']['ohlcv'] + ta.g.config['data']['profit'])
eva_obj.predict(model, self.do.df, x_test)
def download_inx(self, downpath = ta.g.data_path, filename = "inx_code.csv"):
if my_tools.path_exists(filename) == False:
ta.g.log.error("inx file is not exists and exit")
return False
self.checkdir(downpath)
zddown.down_stk_inx(downpath, filename);
return True
def predict(self, mod_filename):
if not my_tools.path_exists(mod_filename):
return
if (self.do == None):
return
self.do.prepared(self.type, action='predict') # 数据预处理
other_features_lst = ta.g.config['data']['ohlcv'] + ta.g.config[
'data']['profit'] # + xagv_lst + ma100_lst + other_lst
x_df = my_dm.util.get_features(self.do.df.tail(5), other_features_lst)
txn = x_df.shape[0]
x_lst = other_features_lst
num_in = len(x_lst)
x_df = x_df.reshape(txn, num_in, -1)
print(x_df)
mo = self.setmod(mod_filename)
y_df = mo.predict(x_df)
print(y_df)
return y_df
def modeling(self, model_filename=""):
if (self.do == None):
return
self.do.prepared(self.type, action='modeling') # 数据预处理
# 分离训练和测试数据
self.df_train, self.df_test = my_dm.util.split(self.do.df, 0.6)
# 构建训练特征数据
other_features_lst = ta.g.config['data']['ohlcv'] + ta.g.config[
'data']['profit'] # + xagv_lst + ma100_lst + other_lst
self.x_train = my_dm.util.get_features(self.df_train,
other_features_lst)
self.x_test = my_dm.util.get_features(self.df_test, other_features_lst)
#############################################################################################################
# 构建特征，也就是结果值Y
if self.type == 'rate':
self.y_train = my_dm.util.prepared_y(self.df_train,
'next_rate_10_type', 'onehot')
self.y_test = my_dm.util.prepared_y(self.df_test,
'next_rate_10_type', 'onehot')
y_lst = self.y_train[0]
x_lst = other_features_lst
num_in, num_out = len(x_lst), len(y_lst)
if self.type == 'price':
self.y_train = my_dm.util.prepared_y(self.df_train,
'next_profit_10')
self.y_test = my_dm.util.prepared_y(self.df_test, 'next_profit_10')
y_lst = 1
x_lst = other_features_lst
num_in, num_out = len(x_lst), y_lst
print('\n self.df_test.tail()', self.df_test.tail())
print('\n self.x_train.shape,', self.x_train.shape)
print('\n type(self.x_train),', type(self.x_train))
rxn, txn = self.x_train.shape[0], self.x_test.shape[0]
self.x_train, self.x_test = self.x_train.reshape(
rxn, num_in, -1), self.x_test.reshape(txn, num_in, -1)
print('\n x_train.shape,', self.x_train.shape)
print('\n type(x_train),', type(self.x_train))
print('\n num_in, num_out:', num_in, num_out)
if not my_tools.path_exists(model_filename):
# mx = zks.rnn010(num_in, num_out)
# mx = zks.lstm010(num_in, num_out)
mx = zks.lstm020typ(num_in, num_out)
else:
mx = self.setmod(model_filename)
mx.summary()
plot_model(mx, to_file=ta.g.log_path + 'model.png')
print('\n#4 模型训练 fit')
tbCallBack = keras.callbacks.TensorBoard(log_dir=ta.g.log_path,
write_graph=True,
write_images=True)
tn0 = arrow.now()
mx.fit(self.x_train,
self.y_train,
epochs=500,
batch_size=512,
callbacks=[tbCallBack])
tn = zt.timNSec('', tn0, True)
self.save(mx, model_filename)
eva_obj = eva.evaluation(self.do)
eva_obj.predict(mx, self.df_test, self.x_test)
def prepared(params):
param_lst = []
if "," in params:
param_lst = params.split(",")
else:
if "|" in params:
param_lst = params.split("|")
else:
param_lst.append(params)
def get_param(param_lst):
type, code, datafile, modfile, lstfile = "rate", "", "", "", ""
if len(param_lst) <= 1:
param = my_tools.params_split(param_lst[0])
param0 = param[0]
if len(param) > 1:
if 'data' == param0:
datafile = param[1]
if 'code' == param0:
code = param[1]
if 'lst' == param0:
lstfile = param[1]
else:
if 'init' in param0.lower():
lstfile = ta.g.config['data']['init_codefile']
else:
if '.csv' in param0.lower():
datafile = param0
else:
code = param0
else:
for j in range(0, len(param_lst)):
param = my_tools.params_split(param_lst[j])
param0 = param[0]
if 'type' == param0:
type = param[1]
if 'code' == param0:
code = param[1]
if 'data' == param0:
datafile = param[1]
if 'mod' == param0:
modfile = param[1]
if 'lst' == param0:
lstfile = param[1]
if len(code) > 0:
code = "%06d" % int(code)
return type, code, datafile, modfile, lstfile
type, code, datafile, modfile, lstfile = get_param(param_lst)
if len(code) > 0 and len(datafile) <= 0: # 有代码没数据文件则先下载
_, datafile = my_dm.download_from_code(code, '2007-01-01')
if len(lstfile) > 0:
if not my_tools.path_exists(lstfile):
lstfile = os.path.join(ta.g.data_path, lstfile)
if not my_tools.path_exists(lstfile):
ta.g.log.error("can't find code file and init it : " + lstfile)
initialize("codefile")
else:
if len(datafile) > 0:
if not my_tools.path_exists(datafile):
datafile = os.path.join(ta.g.stk_path, datafile)
if not my_tools.path_exists(datafile):
ta.g.log.error("can't find data file: " + datafile)
return
if len(code) <= 0 and len(datafile) > 0:
code, _ = my_tools.get_code_from_filename(datafile)
if len(modfile) <= 0 and len(code) > 0:
modfile = ta.g.mod_path + code + ".h5"
return type, code, datafile, modfile, lstfile
def checkdir(self, path): # 创建目录
if my_tools.path_exists(path) == False:
my_tools.mkdir(path)