def __init__(self,
id_lst="",
from_date=datetime.date(1900, 1, 1),
to_date=datetime.date(2200, 12, 31),
do_update=False):
if len(id_lst) > 0:
print("Number of stock_id: %s" % len(id_lst))
self.id_lst = id_lst
self.from_date = from_date
self.to_date = to_date
from_date_str = from_date.strftime("%Y-%m-%d")
to_date_str = to_date.strftime("%Y-%m-%d")
sql_str = "WHERE stock_id in ("
for id_str in self.id_lst:
sql_str = sql_str + "" + str(id_str) + "" + ","
sql_str = sql_str[:-1]
sql_str = sql_str + ")"
sql_str = sql_str + " AND date >= '" + from_date_str + "' AND date <= '" + to_date_str + "'"
sql_str = sql_str + " ORDER BY stock_id, date" # [IMPORTANT]
else:
from_date_str = from_date.strftime("%Y-%m-%d")
to_date_str = to_date.strftime("%Y-%m-%d")
sql_str = "WHERE date >= '" + from_date_str + "' AND date <= '" + to_date_str + "'"
sql_str = sql_str + " ORDER BY stock_id, date" # [IMPORTANT]
self.df = self.crud.read_tbl_by_df(sql_str)
self.id_lst = self.df["stock_id"].drop_duplicates().values
self.id_json_lst = []
for i in self.id_lst:
tmp_json = {
"stock_id":
int(i),
"date_len":
len(self.df.loc[self.df["stock_id"] == int(i), ["date"]])
}
self.id_json_lst.append(tmp_json)
if do_update:
print("Update stock charts...")
for cur_id in self.df["stock_id"].drop_duplicates().values:
try:
start = time.time()
dcs.update_stock(str(cur_id))
scud.update_charts_stats(str(cur_id), True)
elapsed_time = time.time() - start
print("* Update time:{0}".format(elapsed_time) + "[sec]")
except Exception as e:
print("Error occured >>>>> %s" % str(cur_id))
# Tmp code ----------------------------------
self.df = self.df.rename(columns={
'end_price': 'close_price',
'start_price': 'open_price'
})
# Tmp code ----------------------------------
# self.set_sup_res_line(100)
self.set_array()
def get_predict_price_by_models(self, update_fl=False):
# try:
start = time.time()
tolal_cols = ["stock_id", "date", "open_price", "high_price", "low_price", "close_price", "output"]
target_cols = ["open_price", "high_price", "low_price", "close_price", "output"]
s_lst = []
for hs in self.hs_lst:
if update_fl:
dcs.update_stock(hs.stock_id)
scud.update_charts_stats(hs.stock_id)
hs.update_ii_obj(self.d)
s_lst.append(hs.stock_id)
for model in self.models:
tmp_pred_lst = []
r = model.input.shape.dims[1].value
tmp_ii_obj = ii.investing_info(s_lst, self.d-timedelta(days=r*2), self.d-timedelta(days=1))
for hs_id in range(len(self.hs_lst)):
hs = self.hs_lst[hs_id]
sid = hs.stock_id
tmp_df = tmp_ii_obj.df.loc[tmp_ii_obj.df["stock_id"]==int(sid), target_cols]
if len(tmp_df) < r:
continue
# tmp_df = tmp_ii_obj.df.loc[:, tolal_cols]
tmp_np = np.array(df_to_array(tmp_df, target_cols))[:,len(tmp_df)-r:len(tmp_df)]
max_lst = tmp_np.max(axis=1)
for i in range(len(max_lst)):
tmp_np[i, :] = tmp_np[i, :] / max_lst[i]
tmp_x = tmp_np.reshape(len(target_cols), r, -1).T
tmp_predict = model.predict(tmp_x)[0]
# tmp_pred_lst.append(tmp_predict)
for i in range(len(max_lst)):
tmp_predict[i] = tmp_predict[i] * max_lst[i]
tmp_predict_df = pd.Series(index=tolal_cols, dtype=str)
tmp_predict_df["stock_id"] = sid
tmp_predict_df["date"] = self.d
for c in range(len(target_cols)):
tmp_predict_df[target_cols[c]] = tmp_predict[c]
self.predict_price_lst = self.predict_price_lst.append(tmp_predict_df,ignore_index=True)
elapsed_time = time.time() - start
print ("* get_predict_price_by_models time:{0}".format(elapsed_time) + "[sec]")
return self.predict_price_lst
def update_ii_obj(self, d=datetime.today(), updatedb=True):
if updatedb:
dcs.update_stock(self.stock_id)
scud.update_charts_stats(self.stock_id)
self.ii_obj = ii.investing_info([self.stock_id],
d - timedelta(days=365), d, updatedb)
def get_predict_price_by_models(self, update_fl=False):
start = time.time()
predict_lst = []
pre_predict_lst = []
whole_id_lst = []
crud = CRUD_for_STOCK_DETAIL()
for hs in self.hs_lst:
if update_fl:
dcs.update_stock(hs.stock_id)
scud.update_charts_stats(hs.stock_id)
hs.update_ii_obj(self.d)
whole_id_lst.append(hs.stock_id)
m=0
for model in self.models:
tmp_pred_lst = []
r = model.input.shape.dims[1].value
tmp_ii_obj = ii.investing_info(whole_id_lst, self.d-timedelta(days=r*2), self.d-timedelta(days=1))
arr, id_lst = tmp_ii_obj.get_arr_over_range(r, True, prediction_mgr_for_hs_arr.target_cols)
print(arr)
tmp_predict_lst = model.predict(arr)
tmp_arr = np.insert(arr, r, tmp_predict_lst, axis=1)[:, 1:r+1, :]
tmp_pre_predict_lst = model.predict(tmp_arr)
predict_lst = np.append(predict_lst, tmp_predict_lst)
pre_predict_lst = np.append(pre_predict_lst, tmp_pre_predict_lst)
for id_n in range(len(id_lst)):
try:
cur_arr = arr[id_n,:,0:4]
cur_arr = np.insert(cur_arr, len(cur_arr), tmp_predict_lst[id_n, 0:4], axis=0)
cur_arr = np.insert(cur_arr, len(cur_arr), tmp_pre_predict_lst[id_n, 0:4], axis=0)
cur_id = id_lst[id_n]
fig = plt.figure()
d_lst = tmp_ii_obj.df["date"].values[len(tmp_ii_obj.df["date"])-r:len(tmp_ii_obj.df["date"])]
d_lst = np.append(d_lst, [d_lst.max() + timedelta(days=1), d_lst.max() + timedelta(days=2)])
xdate = [x for x in d_lst] # 日付
ax = plt.subplot()
ax.set_xlim(d_lst[0], d_lst[-1])
ax.grid()
stock_name = crud.read_tbl_by_df("WHERE stock_id='%s'"%str(cur_id))["stock_name"].values[0]
ax.set_title('Chart [%s]_%s'%(str(cur_id),stock_name))
ohlc = np.vstack((date2num(xdate), cur_arr.T)).T
candlestick_ohlc(ax, ohlc, width=0.7, colorup='r', colordown='b')
if not(os.path.isdir("./pred_charts/%s"%str(cur_id))):
os.makedirs("./pred_charts/%s"%str(cur_id))
plt.savefig(os.path.join("./pred_charts/%s"%str(cur_id), '%s_[%s_%s]_%s.png'%(str(cur_id), str(d_lst[0]), str(d_lst[-1]), str(m))))#画像保存
plt.close()
except Exception as e:
import traceback
traceback.print_exc()
m = m + 1
predict_lst = predict_lst.reshape(len(self.models), len(id_lst), -1)
pre_predict_lst = pre_predict_lst.reshape(len(self.models), len(id_lst), -1)
elapsed_time = time.time() - start
print ("* get_predict_price_by_models time:{0}".format(elapsed_time) + "[sec]")
return predict_lst, pre_predict_lst
def test_plot(s_lst, init_date, r):
import numpy as np
import pandas as pd
import investing_info as ii
from keras.models import load_model
from datetime import datetime, timedelta, date
from mpl_finance import candlestick_ohlc
from matplotlib.dates import date2num
import matplotlib.pyplot as plt
import os
import download_charts_selenium as dcs
import stock_charts_updateDB as scud
for cur_id in s_lst:
dcs.update_stock(cur_id)
scud.update_charts_stats(cur_id)
m1=load_model("./tmp_30/models/full/model.ep50.h5")
m2=load_model("./tmp_30/models/Tosho1bu/model.ep50.h5")
m3=load_model("/Volumes/StreamS06_2TB/project_a/tmp_90/models/Tosho1bu/model.ep29.h5")
pred_price_lst = pd.DataFrame()
# init_date = datetime(2018,8,5)
for i in range(r):
from_d=datetime(2010,1,1)
to_d=init_date - timedelta(days=r-i)
ii_obj = ii.investing_info(s_lst, from_d, to_d)
# ii_obj = ii.get_ii_obj_by_stock_market("東証一部", from_d, to_d)
# p = prediction_mgr_for_ii(ii_obj, [m1])
p = prediction_mgr_for_ii(ii_obj, [m1, m2, m3])
tmp_pred_price_lst = p.get_predict_price_by_models()
pred_price_lst = pred_price_lst.append(tmp_pred_price_lst)
id_lst = pred_price_lst["stock_id"].drop_duplicates()
for cur_id in id_lst:
full_df_tmp = pred_price_lst.loc[pred_price_lst["stock_id"]==cur_id, :]
# print(full_df_tmp)
fig = plt.figure()
# For pred graph =======
full_df_tmp = full_df_tmp.groupby("date").mean().sort_values(by=["date"], ascending=True)
date_lst1 = pred_price_lst["date"].drop_duplicates()
xdate1 = [x for x in date_lst1] # 日付
ax1 = plt.subplot(211)
df_tmp1 = full_df_tmp.loc[:, ['open_price', 'high_price','low_price', 'close_price']]
ax1.set_xlim(date_lst1.values[0], date_lst1.values[-1]) # x軸の範囲
ax1.grid() # グリッド表示
ax1.set_title('Predicted_plot')
ochl1 = np.vstack((date2num(xdate1), df_tmp1.values.T)).T
candlestick_ohlc(ax1, ochl1, width=0.7, colorup='g', colordown='r')
# fig.autofmt_xdate() # x軸のオートフォーマット
# plt.savefig(os.path.join("./pred_real_charts", str(cur_id) + '_pred.png'))#画像保存
# fig2 = plt.figure()
# For real graph =======
tmp_ii_obj = ii.investing_info([cur_id], date_lst1.values[0], date_lst1.values[-1])
date_lst2 = tmp_ii_obj.df["date"].drop_duplicates()
xdate2 = [x for x in date_lst2] # 日付
ax2 = plt.subplot(212)
df_tmp2 = tmp_ii_obj.df.loc[:, ['open_price', 'high_price','low_price', 'close_price']]
ax2.set_xlim(tmp_ii_obj.df.loc[:,"date"]. values[0], tmp_ii_obj.df.loc[:,"date"]. values[-1]) # x軸の範囲
ax2.grid() # グリッド表示
ax2.set_title('Real_plot')
ochl2 = np.vstack((date2num(xdate2), df_tmp2.values.T)).T
candlestick_ohlc(ax2, ochl2, width=0.7, colorup='g', colordown='r')
# fig.autofmt_xdate() # x軸のオートフォーマット
fig.tight_layout() # タイトルとラベルが被るのを解消
plt.savefig(os.path.join("./pred_real_charts", str(cur_id) + '_real.png'))#画像保存