def sell(self, code, unit_price=0, cnt=0, desc=""):
kl = self.kls[code]
data = kl.get_kabuka()
(indexes, dates, open, high, low, close, volume) = self.data
if self.stocks.has_key(code) == False:
return False
if unit_price == 0:
unit_price = close[indexes[self.date]]
if cnt == 0:
cnt = self.stocks[code].get_cnt()
s = self.stocks[code]
buy_price = s.get_buy_price()
price = unit_price * cnt
pm = price - buy_price * cnt
ret = s.sell(unit_price, cnt)
if ret < 0:
return False
if ret == 0:
del self.stocks[code]
self.balance = self.balance + price - kf.get_charge(price)
f.log(
"[sell] C=%s, UP=%d, CNT=%d, P=%d, B=%d, PM=%d: R=[%s]"
% (code, unit_price, cnt, price, self.balance, pm, desc)
)
def learn2(prgname, meigaras=[]):
f.log("Starting learning")
rootd = f.get_json_data(prgname)
pf = get_element("play_fields", rootd)
# Generate PlayField Data
if pf["enabled"] == 1:
mpf.create_pf2(prgname, meigaras)
# Generate PlayField Data
if pf["enabled"] == 1:
mpf.create_pf(prgname, meigaras)
# Train Players
tp = get_element("train_player", rootd)
if tp == None:
return
if tp["enabled"] == 1:
startd = tp["startd"]
endd = tp["endd"]
tm = TraderMemory(meigaras, startd, endd)
#mtp.train(prgname, tm)
trn = Training(prgname, tm)
trn.train()
f.log("Finished learning")
def _check_forecast_proc(self, datei, mode="DAILY"):
if self.use_forecast == False:
return (DIR_UNKNOWN, 0)
datei -= 1
if self.forec_cache.has_key(datei):
return self.forec_cache[datei]
starti = datei - self.peak_check_period + 1
results = []
ximages = self.tfl.get_xs(self.code, self.data, starti, datei)
if len(ximages) > 0:
results = self.tfl.predict(ximages)
(open, high, low, close, volume) = self._get_prices(datei+1, self.peak_check_period+2)
for i in reversed(range(len(results))):
points = results[-i]
max_point = max(points)
for j in range(len(points)):
if points[j] > TRUSTED_POINTS and points[j] == max_point:
if j == 0 and min(low) >= low[-2-i]:
if DEBUG and self.curr_holding_n > 0:
f.log("[%s] Bottom peak. Forecast=Up" % self.code)
self.forec_cache[datei] = (DIR_UP, points[j])
return self.forec_cache[datei]
if j == 2 and max(high) <= high[-2-i]:
if DEBUG and self.curr_holding_n > 0:
f.log("[%s] Top peak, Forecast=Down" % self.code)
self.forec_cache[datei] = (DIR_DOWN, points[j])
return self.forec_cache[datei]
self.forec_cache[datei] = (DIR_UNKNOWN, 0)
return self.forec_cache[datei]
def g_copy_history(self, endclock):
lastclock = self._get_lastcopyclock()
startclock = cf.clockaddminutes(endclock, -D_HISTORY_COPY_CLOCK_RANGE)
if lastclock <= 0:
lastclock = startclock
if startclock > lastclock and lastclock > 0:
endclock2 = cf.clockaddminutes(lastclock, +D_HISTORY_COPY_CLOCK_RANGE*2)
endclock = min(endclock, endclock2)
startclock = startclock
condlist = ["itemid in (%s)" % ",".join(cf.list2strlist(self._itemids))]
condlist.append("clock >= %s" % str(startclock))
condlist.append("clock <= %s" % str(endclock))
strwhere = sf.list2where(condlist)
strsql = "select itemid, clock, value from history %s;" % (strwhere)
if DEBUG:
print strsql
cf.log(strsql)
g_zbx = sf.g_get_data(MYSQL_ZBX, strsql)
strsql = ""
for row in g_zbx:
(itemid, clock, value) = row
yield row
strsql = "%sreplace into history (itemid, clock, value) values(%s,%s,%s);\n" \
% (strsql, itemid, clock, value)
sf.exec_updsql(MYSQL_JUBATUS, strsql)
self._upd_lastcopyclock(endclock)
def _run_anomaly(self, endclock, method="add"):
# Prepare data
if method == "add":
g_data = self.dgetter.g_get_history(endclock)
if method == "calc":
g_data = self.dgetter.g_copy_history(endclock)
datadict = {}
for row in g_data:
(itemid, clock, value) = row
itemid = str(itemid)
if datadict.has_key(itemid):
self._juba_proc(clock, datadict, method)
datadict = {}
if not datadict.has_key(self.hostid):
(hour, weekday) = self.expand_clock(clock)
datadict["hostid"] = self.hostid
datadict["weekday"] = weekday*1.0/7
datadict["hour"] = hour*1.0/24
datadict[itemid] = value
if len(datadict) > 0:
self._juba_proc(clock, datadict, method)
if method=="add":
cf.log("Saving learned model")
self.anom.save("latest")
self.dgetter.remove_history(endclock)
def learn(prgname, meigaras=[]):
f.log("Starting learning")
tm = TraderMemory(meigaras, startd, endd)
trn = Training(prgname, tm)
trn.train()
f.log("Finished learning")
def restore(self, ckptfile):
if os.path.exists(ckptfile):
saver = tf.train.Saver()
saver.restore(self.sess, ckptfile)
cf.log("Loaded from %s" % (ckptfile))
else:
cf.log("Ckpt file %s does not exists." % (ckptfile))
def _lm2trader(self):
meigaras = self.traders.keys()
if len(meigaras) >= self.division:
f.log("Already holds %d stocks." % len(meigaras))
for i in range(0, self.division):
if len(meigaras) == 0:
wherestr = ""
else:
wherestr = "where %s" % (kf.where_code_in(meigaras, True))
strsql = "select name, code, fieldid, geneid, points, holding_days, trade_mode \
from trade.learned_memory %s order by points desc limit 100;" % (wherestr)
data = sql.exec_selsql(strsql)
if len(data) == 0:
f.log("No good meigaras to get this day.")
break
for row in data:
i = iter(row)
name = next(i)
code = next(i)
fieldid = next(i)
geneid = next(i)
points = next(i)
holding_days = int(next(i))
trade_mode = next(i)
if self.traders.has_key(code) == False:
return Trader(code, geneid, self.tm, trade_mode, holding_days,
"name=%s, fieldid=%s, geneid=%s, points=%s, holding_days=%s, trade_mode=%s" \
% (name, fieldid, geneid, str(points), str(holding_days), trade_mode))
def restore(self, load_dir="", current_date=""):
ver = -1
if self.is_restored:
return (False, "", 0)
self.set_params(save_folder=load_dir)
if os.path.exists(self.version_file):
try:
with open(self.version_file, "r") as f:
line = (f.read()).split(",")
ver = line[0]
self.train_accuracy = line[1]
f.close()
if current_date != "":
if ver >= self.startd:
cf.log("Ckpt file is newer than current date.")
return (False, ver, self.train_accuracy)
if ver < dtf.datestrsub(current_date, CKPT_VALID_DAYS):
cf.log("Ckpt file is too old.")
return (False, ver, self.train_accuracy)
saver = tf.train.Saver()
saver.restore(self.sess, self.ckptfile)
cf.log("Loaded from %s" % (self.ckptfile))
self.is_restored = True
return (True, ver, self.train_accuracy)
except Exception as e:
cf.log("Failed to load")
raise
else:
cf.log("Ckpt file %s does not exists." % (self.ckptfile))
return (False, ver, self.train_accuracy)
def _calc_score_proc(self, geneid):
self.parameter = -1
self.fieldids = self.coaches.keys()
self._alloc_players(geneid)
self._play_proc(self.fieldids)
self._calc()
self._selection()
f.log("Finished playing")
def log(self, msg):
if self.logfile == "":
f.log(msg)
else:
logdir = "%s/simlator3" % LOG_DIR
if os.path.exists(logdir) == False:
os.makedirs(logdir)
f.log(msg, self.logfile, logdir)
def learn_line(prgname, meigaras=[]):
f.log("Starting learning")
rootd = f.get_json_data(prgname)
pf = get_element("play_fields", rootd)
# Generate PlayField Data
if pf["enabled"] == 1:
mpf.create_pf2(prgname, meigaras)
def train(self):
for i in range(TRAIN_NUM):
# batch = mnist.train.next_batch(25)
batch = self.get_rnd_lines(BATCH_SIZE)
if i % 10 == 0:
train_accuracy = self.accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})
cf.log("step %d, training accuracy %g" % (i, train_accuracy))
self.train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
cf.log("Finished training")
def run(self):
for i in range(50):
batch = mnist.train.next_batch(25)
if i%10 == 0:
train_accuracy = self.accuracy.eval(feed_dict={
x:batch[0], y_: batch[1], keep_prob: 1.0})
cf.log("step %d, training accuracy %g"%(i, train_accuracy))
self.train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
cf.log("Finished training")
def plot(self, startd, endd):
(indexes, dates, open, high, low, close, volume) = self.data
startd = self._get_near_date(startd, "f", indexes)
endd = self._get_near_date(endd, "b", indexes)
if startd == endd:
f.log("plot:startd and endd is the same for code %s" % self.code)
starti = indexes[startd]+60
endi = indexes[endd]
(xsl, ysl, labelsl) = self._get_xy(starti, endi, dates)
plt.plot_kabuka2png(xsl, ysl, labelsl, "%s/%s_%s-%s.png" % (TMP_DIR, self.code, startd, endd))
def __init__(self, meigaras, startd="", endd=""):
'''
Constructor
'''
self.startd = startd
self.endd = endd
self.normal_lines = {}
self.base_lines = {}
self.trend_lines = {}
self.data = kf.get_kabuka(meigaras, startd, endd)
self._make_lines()
f.log("finished preparation")
def train(self, train_num=0):
if train_num == 0:
train_num = TRAIN_NUM
for i in range(train_num):
#batch = mnist.train.next_batch(25)
batch = self.get_line_data()
if i%10 == 0:
train_accuracy = self.accuracy.eval(feed_dict={
self.x:batch[0], self.y_: batch[1], self.keep_prob: 1.0})
cf.log("step %d, training accuracy %g"%(i, train_accuracy))
self.train_step.run(feed_dict={self.x: batch[0], self.y_: batch[1], self.keep_prob: 0.5})
cf.log("Finished training")
def buy(self, code, unit_price, cnt, desc=""):
price = unit_price * cnt
tmp_balance = self.balance - price - kf.get_charge(price)
if tmp_balance < 0:
return False
if self.stocks.has_key(code):
self.stocks[code].buy(unit_price, cnt)
else:
self.stocks[code] = StockInfo2(code, self.date, self.kls[code], unit_price, cnt)
self.balance = tmp_balance
f.log("[buy] C=%s, UP=%d, CNT=%d, P=%d, B=%d: R=[%s]" % (code, unit_price, cnt, price, self.balance, desc))
return True
def create_play_field(startd, endd, meigaras=[]):
f.log("Starting to create play field")
(mcode, dl, X, y, report) = gen_pf_line("20150601", "20160101", meigaras)
mcode = np.array(mcode)
dl = np.array(dl)
X = np.array(X)
tablename = "pf_line"
tbl.create_table_from_tmpl("pf2", [tablename])
r = np.c_[mcode, dl, X]
tbl.arr2table(r, tablename)
f.log("Finished creating play field")
def _set_dicts(self):
odb = db()
cursor = odb.get_cursor()
#strsql = "select code, date, close, volsize, dow_sign from trade.dow "
strsql = "select code, date, close from trade.kabuka "
wherestr = kf.general_kabuka_where(self.meigaras, self.startd, self.endd)
strsql = strsql + wherestr
strsql = strsql + " order by code, date;"
try:
cursor.execute(strsql)
except MySQLdb.Error, e:
f.log(strsql)
raise Exception("SQL ERROR %d: %s" % (e.args[0], e.args[1]))
def train_bydate(cnt=100):
json = f.get_json_data("ml_train_tf")
span = json["span"]
term_len = json["term_len"]
dup_len = json["dup_len"]
train_startd = span[0]
train_endd = span[1]
starti = 0
eigyobis = dtf.get_eigyobis(train_startd, train_endd)
while starti+term_len <= len(eigyobis):
tmp_endd = eigyobis[starti+term_len]
f.log("Training nikkei 225")
train225(eigyobis[starti], tmp_endd)
f.log("Training from %s to %s" % (eigyobis[starti], tmp_endd))
multi_training(eigyobis[starti], tmp_endd, get_meigaras(cnt, tmp_endd))
predict_starti = starti+term_len+1
if predict_starti < len(eigyobis):
predict_endi = predict_starti + term_len
if predict_endi >= len(eigyobis):
predict_endi = len(eigyobis)-1
tmp_endd = eigyobis[predict_endi]
f.log("Making prediction from %s to %s" % (eigyobis[predict_starti], tmp_endd))
multi_predict2db(eigyobis[starti], tmp_endd, get_meigaras(cnt, tmp_endd, term_len))
starti += term_len - dup_len
f.log("Finished")
def g_get_history(self, endclock):
itemids = self._itemids
startclock = cf.clockaddminutes(endclock, -LEARN_PERIOD)
condlist = ["itemid in (%s)" % ",".join(cf.list2strlist(itemids))]
condlist.append("clock >= %d" % int(startclock))
condlist.append("clock <= %d" % int(endclock))
strwhere = sf.list2where(condlist)
# Get history data from Zabbix
strsql = "select itemid, clock, value from history %s order by clock, itemid;" % (strwhere)
if DEBUG:
cf.log(strsql)
g_data = sf.g_get_data(MYSQL_JUBATUS, strsql)
for row in g_data:
(itemid, clock, value) = row
yield (itemid, clock, value)
def _juba_proc(self, clock, datadict, method="add"):
#if DEBUG:
# print datadict
datum = Datum()
for k in datadict.keys():
#print "key:%s value:%s" % (str(k), str(datadict[k]))
if k == "hostid":
datum.add_number(str(k), int(datadict[k])*1.0/ZBX_ITEMID_DIGITS)
elif k == "weekday" or k == "hour":
datum.add_number(str(k), datadict[k])
elif k != "hostid" and k != "weekday" and k != "hour":
datum.add_number(str(k), self.norm(k, datadict[k]))
#print datum
retry_cnt = JUBA_RETRY_MAX
while True:
try:
if method=="add":
print datum
ret = self.anom.add(datum)
exit()
if method=="calc":
print datum
score = self.anom.calc_score(datum)
if score == float('Inf') or score > ML_LIMIT:
#print datadict
if self.alarm_on == False:
self.alarm_on = True
cf.log("[%s] score=%f" % (cf.clock2strjst(clock), score))
else:
if self.alarm_on == True:
self.alarm_on = False
cf.log("[%s] score recovered to normal:score=%f" % (cf.clock2strjst(clock), score))
break
except (msgpackrpc.error.TransportError, msgpackrpc.error.TimeoutError) as e:
retry_count -= 1
if retry_count <= 0:
raise
self.anom.get_client().close()
self.set_anom()
print e
time.sleep(JUBA_RETRY_INTERVAL)
continue
def _train_proc(self):
self.fieldids = self.coaches.keys()
f.log(" Number of fields: %d" % len(self.fieldids))
f.log(" Allocating first players")
self._alloc_players_from_db()
self._alloc_default_players()
self._alloc_random_players()
for i in range(0, self.train_count+1):
f.log(" Processing %d turn.." % (i))
random.shuffle(self.fieldids)
self._mating()
self._copy()
self._mutate()
self._alloc_random_players()
f.log(" Training..")
self._play()
f.log(" Selection..")
self._selection()
def train(self):
f.log("Start training..")
self._set_params()
# Generate coaches for each play fields
f.log("Getting coaches")
self._get_coaches(self.pf_table)
# Create table if not exists
tbl.create_table_from_tmpl(self.tp_template, [self.tp_table])
# Train
f.log("Training with GA")
self._train_proc()
# Insert report into table
f.log("Creating report")
self._report()
f.log("Finished training..")
def simulate(self):
datelist = dtf.get_datestr(self.startd, self.endd)
self.relative_date_idx = 0
for d in datelist:
self._set_date(d)
trader = self.choose(self.traders.keys())
if trader is not None:
code = trader.get_code()
if not self.traders.has_key(code):
self.traders[code] = trader
for code in self.traders.keys():
self.trade(code)
(eval_price, holding_n) = self.get_eval()
#self._sell_all()
f.log("Final evaluate amount is " + str(eval_price))
f.log("Finished simulation.")
def simulate(self):
datelist = dtf.get_datestr(self.startd, self.endd)
self.relative_date_idx = 0
for d in datelist:
self._set_date(d)
self.sell()
trader = self.choose(self.meigaras, self.traders.keys())
self.buy(trader)
self.relative_date_idx += 1
if self.relative_date_idx == self.one_cycle_period and self.one_cycle_period > 0:
self.relative_date_idx = 0
eval_price = self.account.get_eval_price()
init_amount = self.account.get_init_amount()
#self._sell_all()
f.log("Final evaluate amount is " + str(eval_price))
f.log("Finished simulation.")
def check_forecast(self, datei):
if self.use_forecast == False:
return (DIR_UNKNOWN, 0)
datei -= 1
if self.forec_cache.has_key(datei):
return self.forec_cache[datei]
starti = datei - self.peak_check_period + 1
(indexes, dates, open, high, low, close, volume) = self.data
#self.tfl.train(11)
#ximage = self.tfl.get_xs(self.code, self.data, starti, datei)
#results = tff.predict(self.code, self.data, starti, datei, self.code)
results = []
for d in range(starti, datei+1):
predict = self.get_prediction(dates[d])
if predict.has_key("last_update") == False:
continue
last_update = predict["last_update"]
if last_update <= dates[datei]:
results.append(predict["value"])
(open, high, low, close, volume) = self._get_prices(datei, self.peak_check_period+1)
for i in reversed(range(len(results))):
points = results[-i]
if len(points) == 0:
continue
max_point = max(points)
for j in range(len(points)):
if points[j] > TRUSTED_POINTS and points[j] == max_point:
if j == 0 and min(low) >= low[-2-i] and low[0] > low[-2-i]:
if DEBUG and self.curr_holding_n > 0:
f.log("[%s] Bottom peak. Forecast=Up" % self.code)
self.forec_cache[datei] = (DIR_UP, points[j])
return self.forec_cache[datei]
if j == 2 and max(high) <= high[-2-i] and high[0] < high[-2-i]:
if DEBUG and self.curr_holding_n > 0:
f.log("[%s] Top peak, Forecast=Down" % self.code)
self.forec_cache[datei] = (DIR_DOWN, points[j])
return self.forec_cache[datei]
self.forec_cache[datei] = (DIR_UNKNOWN, 0)
return self.forec_cache[datei]
def check_forecast(self, datei):
if self.use_forecast == False:
return DIR_UNKNOWN
datei -= 1
starti = datei - self.peak_check_period + 1
ximage = self.tf_learning.get_xs(self.code, self.data, starti, datei)
results = self.tf_learning.predict(ximage)
for i in reversed(range(len(results))):
points = results[-i]
max_point = max(points)
for j in range(len(points)):
if points[j] > TRUSTED_POINTS and points[j] == max_point:
if j == 0:
if DEBUG and self.curr_holding_n > 0:
f.log("[%s] Bottom peak. Forecast=Up" % self.code)
return DIR_UP
if j == 2:
if DEBUG and self.curr_holding_n > 0:
f.log("[%s] Top peak, Forecast=Down" % self.code)
return DIR_DOWN
return DIR_UNKNOWN
def simulate(dates):
dates.sort()
for i in range(1, len(dates)):
startd = dates[i-1]
endd = dates[i]
files = glob.glob("%s/20*" % PNFCKPT_DIR)
files.sort()
ckptdir = ""
for f in reversed(files):
if os.path.isdir(f):
foldername = f.split("/")[-1]
if startd >= foldername:
ckptdir = foldername
break
if ckptdir == "":
f.log("No learned data")
exit(1)
meigaras = kf.get_goodmeigaras(endd, 20)
s = Simulator3(meigaras, "BOTH", "%s.log" % ckptdir, ckptdir)
s.simulate(startd, endd)
s = None
