def load_bitcoin_ohlc(code, start_date, end_date, with_filter=True, strict=False, time=None):
start = utils.to_datetime(start_date)
end = utils.to_datetime(end_date)
current = start
data = None
while current <= end:
current = utils.to_datetime(utils.to_format(current + relativedelta(days=1), output_format="%Y-%m-%d"))
try:
data_ = pandas.read_csv(Loader.bitcoin_dir + "/" + str(code) + "/" + utils.to_format(current) + ".csv")
except:
continue
data = data_ if (data is None) else pandas.concat([data, data_])
if data is None:
return None
data = Loader.format(data, float, date_format="%Y-%m-%d %H:%M:%S")
data["low"] = data[["low", "open", "close"]].apply(min, axis=1)
data["high"] = data[["high", "open", "close"]].apply(max, axis=1)
if with_filter:
end_time = "23:59:59" if time is None else utils.format("%s %s" % (end_date, time), output_format="%H:%M:%S")
filtered = Loader.filter(data, "%s 00:00:00" % start_date, "%s %s" % (end_date, end_time), strict)
if len(filtered) == 0:
return None
return filtered
return data
return data
def target_codes(args, terms, strategy_simulator):
codes = []
for term in terms:
start = utils.to_format(term["start_date"])
end = utils.to_format(term["end_date"])
targets = select_codes(args, start, end, strategy_simulator)
codes = list(set(codes + targets))
return codes
def load_portfolio(date, price, length=10, by_day=False):
if by_day:
d = utils.to_format(utils.select_weekday(date, to_before=False))
else:
d = utils.to_format(datetime(date.year, date.month, 1))
try:
data = pandas.read_csv("portfolio/financials/%s.csv" % d, header=None)
data.columns = ["code", "value", "count", "date"]
data = data.iloc[:length]
except:
data = None
return data
def load_portfolio(signals, date, price, length=10, by_day=False):
if by_day:
d = utils.to_format(utils.select_weekday(date, to_before=False))
else:
d = utils.to_format(datetime(date.year, date.month, 1))
try:
data = pandas.read_csv("portfolio/%s/%s.csv" % (signals, d))
data = data[data["price"] <= price]
data = data.iloc[:length]
except:
data = None
return data
def simulate_params(stocks, terms, strategy_simulator, ignore_manda=True):
params = []
strategy_simulator.simulator_setting.strategy = None
strategy_creator = strategy_simulator.strategy_creator(stocks["args"])
for term in terms:
start = utils.to_format(term["start_date"])
end = utils.to_format(term["end_date"])
codes = select_codes(stocks["args"], start, end, strategy_simulator)
select = select_data(codes, stocks, start, end, strategy_creator)
params.append((select, utils.to_format(utils.to_datetime(start)), end,
ignore_manda))
# print("simulate params:", start, end, utils.timestamp())
return params
def load_tick(code,
start_date,
end_date,
with_filter=True,
strict=False,
time=None):
start = utils.to_datetime(start_date)
end = utils.to_datetime(end_date)
current = start
data = None
while current <= end:
month = current.strftime("%Y%m")
current = utils.to_datetime(
utils.to_format(current + relativedelta(months=1),
output_format="%Y-%m-01"))
try:
data_ = pandas.read_csv(Loader.tick_dir + "/" + month + "/" +
str(code) + ".csv",
header=None)
except:
continue
data = data_ if (data is None) else pandas.concat([data, data_])
if data is None:
return None
data = Loader.tick_format(data)
if with_filter:
end_time = "23:59:59" if time is None else utils.format(
"%s %s" % (end_date, time), output_format="%H:%M:%S")
filtered = Loader.filter(data, "%s 00:00:00" % start_date,
"%s %s" % (end_date, end_time), strict)
if len(filtered) == 0:
return None
return filtered
return data
def before_ranking(date, ranking_type, before=1):
d = utils.to_datetime(date) - utils.relativeterm(before, with_time=True)
while not utils.is_weekday(d):
d = d - utils.relativeterm(1, with_time=True)
d = utils.to_format(d)
stocks = Loader.ranking(d, ranking_type)
return stocks
def load_portfolio(date, price, length=10, by_day=False):
if by_day:
d = utils.to_format(utils.select_weekday(date, to_before=False))
else:
d = utils.to_format(datetime(date.year, date.month, 1))
try:
data = pandas.read_csv("portfolio/high_update/%s.csv" % d, header=None)
data.columns = ["code", "price", "count", "date"]
data = data[data["price"] <= price]
if len(data) < 50 or len(data) > 550:
data = None
else:
data = data.iloc[:length]
except:
data = None
return data
def get_targets(self, args, targets, date, use_cache=False):
if args.code is None:
if args.instant:
date = utils.to_format(utils.select_weekday(utils.to_datetime(date), to_before=False))
targets = list(self.strategy_creator(args, use_cache=use_cache).subject(date))
else:
targets = [args.code]
return targets
def load_portfolio(date, price, length=10):
d = utils.to_format(datetime(date.year, date.month, 1))
try:
data = pandas.read_csv("portfolio/financials/%s.csv" % d, header=None)
data.columns = ["code", "value", "count", "date"]
data = data.iloc[:length]
except:
data = None
return data
def load_portfolio(date, length=10):
d = utils.to_format(date)
try:
data = pandas.read_csv("portfolio/per/%s.csv" % d, header=None)
data.columns = ["code", "per", "close"]
data = data.iloc[:length]
except:
data = None
return data
def select_codes(self, args, start_date, end_date):
codes = []
dates = self.strategy_creator(args, use_cache=True).select_dates(start_date, end_date, instant=args.instant)
for d in dates:
date = utils.to_format(d)
targets = self.get_targets(args, codes, date, use_cache=True)
codes = list(set(codes + targets))
return codes
def load_portfolio(date, price, length=10):
d = utils.to_format(datetime(date.year, date.month, 1))
try:
data = pandas.read_csv("portfolio/new_high/%s.csv" % d, header=None)
data.columns = ["code", "price"]
data = data[data["price"] <= price]
data = data.iloc[:length]
except:
data = None
return data
def load_portfolio(signals, date, price, length=10):
# d = utils.to_format(datetime(date.year, date.month, 1))
d = utils.to_format(date)
try:
data = pandas.read_csv("portfolio/%s/%s.csv" % (signals, d))
data = data[data["price"] <= price]
data = data.iloc[:length]
except:
data = None
return data
def load_portfolio(self, date):
d = utils.to_format(datetime(date.year, date.month, 1))
try:
data = pandas.read_csv("portfolio/high_update/%s.csv" % d,
header=None)
data.columns = ["code", "price", "count", "date"]
data = data[data["price"] <= (self.setting.assets / 250)]
data = data.iloc[:10]
except:
data = None
return data
def load_stock(self, start_date, end_date):
combination_setting = strategy.CombinationSetting()
strategy_creator = CombinationStrategy(combination_setting)
codes = []
for date in utils.daterange(utils.to_datetime(start_date),
utils.to_datetime(end_date)):
codes = list(
set(codes + strategy_creator.subject(utils.to_format(date))))
data = None
while data is None or len(data) <= self.state_size:
self.code = numpy.random.choice(codes)
data = Loader.load(
self.code,
utils.to_format(
utils.to_datetime(start_date) - relativedelta(months=12)),
end_date)
data = utils.add_stats(data)
data = Loader.filter(data, start_date, end_date)
print("code: [%s]" % self.code)
return data
def validation(args, stocks, terms, strategy_simulator, combination_setting,
strategy_settings):
performances = {}
strategy_simulator.strategy_settings = strategy_settings
strategy_simulator.combination_setting = combination_setting
params = simulate_params(stocks,
terms,
strategy_simulator,
ignore_manda=args.ignore_manda)
simulator_setting = copy.deepcopy(strategy_simulator.simulator_setting)
if args.verbose or args.apply_compound_interest:
print("debug mode")
for param in params:
_, start_date, end_date, _ = param
result = simulate_by_term((strategy_simulator,
strategy_settings[-1]) + param)
if args.apply_compound_interest: # 複利を適用
strategy_simulator.simulator_setting.assets += result["gain"]
performances[utils.to_format(
utils.to_datetime_by_term(end_date))] = result
else:
params = list(
map(lambda x: (strategy_simulator, strategy_settings[-1]) + x,
params))
results = simulate_by_multiple_term(stocks, params)
for result in results:
performances[utils.to_format(
utils.to_datetime_by_term(result["end_date"]))] = result
# 検証スコア
score = -evaluator.get_score(args, performances.values(),
strategy_simulator.simulator_setting,
strategy_settings[-1])
# 結果の表示 =============================================================================
report = create_performance(args, simulator_setting, performances)
return score, report, performances
def load_portfolio(self, date, length=10):
d = utils.to_format(date)
try:
data = pandas.read_csv("portfolio/high_update/%s.csv" % d,
header=None)
data.columns = ["code", "price", "count", "date"]
data = data[data["price"] <= (self.setting.assets / 500)]
if len(data) < 50 or len(data) > 550:
data = None
else:
data = data.iloc[:length]
except:
data = None
return data
def select_data(codes, stocks, start, end, strategy_creator):
select = {"data": {}, "index": stocks["index"], "args": stocks["args"]}
args = select["args"]
for code in codes:
if not code in stocks["data"].keys():
continue
start_date = utils.to_format(
utils.to_datetime(start) - utils.relativeterm(3))
select["data"][code] = stocks["data"][code].split(start_date, end)
select["data"][code] = strategy_creator.add_data(
stocks["data"][code], stocks["index"])
return select
def steady_trend_stocks(start_date, end_date, filename="rising_stocks.csv"):
current = start_date
data = {"all": []}
output_format = "%Y%m"
while int(utils.format(current, output_format=output_format)) <= int(utils.format(end_date, output_format=output_format)):
try:
months = utils.format(current, output_format=output_format)
d = pandas.read_csv("%s/steady_trend_stocks/%s/%s" % (Loader.settings_dir, months, filename), header=None)
d.columns = ['code']
data[months] = d["code"].values.tolist()
data["all"] = list(set(data["all"] + data[months]))
except:
continue
finally:
current = utils.to_format(utils.to_datetime(current) + utils.relativeterm(1))
return data
def load(args, codes, terms, combination_setting):
min_start_date = min(list(map(lambda x: x["start_date"], terms)))
prepare_term = utils.relativeterm(args.validate_term)
start_date = utils.to_format(min_start_date - prepare_term)
end_date = utils.format(args.date)
strategy_creator = strategy.load_strategy_creator(args,
combination_setting)
print("loading %s %s %s %s" %
(len(codes), min_start_date, start_date, end_date))
index = strategy.load_index(args, start_date, end_date)
data = {}
params = list(
map(
lambda x: {
"code": x,
"start_date": start_date,
"end_date": end_date,
"args": args
}, codes))
try:
p = Pool(args.jobs)
ret = p.map(create_simulator_data, params)
for r in ret:
if r is None:
continue
data[r.code] = r
except KeyboardInterrupt:
p.close()
exit()
finally:
p.close()
print("loading done")
return {"data": data, "index": index, "args": args}
def apply_trace(figure, before, date, code, row, col, rows, cols, layout,
options):
args = create_args(code)
end = date
start = utils.to_format(
utils.to_datetime(end) - utils.relativeterm(int(before)))
simulator_data = strategy.load_simulator_data(
args.code,
start,
end,
args,
names=["average"] if options["use_multi"] else [])
df = simulator_data.middle
df = df[df["date"] >= start]
df = df.reset_index()
# 陽線-> candle.data[1], 陰線 -> candle.data[0]
candle = FF.create_candlestick(df["open"],
df["high"],
df["low"],
df["close"],
dates=df["date"])
stocks = list(candle.data)
figure = add_stats(figure, row, col, stocks, df, []) # ローソク足をグラフに適用
# 日付リスト
dates = list(map(lambda x: utils.to_datetime(x), df["date"].astype(str)))
rangebreaks = [{
"values":
list(
filter(lambda x: not utils.is_weekday(x),
utils.daterange(dates[0], dates[-1])))
}] # 休日を除外
daterange = [dates[0], dates[-1]]
layout_settings = create_layout_settings(options)
if options["use_multi"]:
setting = layout_settings[0]
setting["line"] = [setting["line"][0]] # candle with average
layout_settings = [setting]
# ===================================================================
# layout_settingをもとに各グラフにデータを入れる
domain_from = 0
for setting in sorted(layout_settings, key=lambda x: x["to"]):
i = row if options["use_multi"] else setting["id"]
label_i = (
(col - 1) +
(row - 1) * cols) + 1 if options["use_multi"] else setting["id"]
# multiの場合同じlegendがいくつも並ぶので最初のだけ有効に
showlegend = layout["showlegend"] and row == 1 and col == 1 if options[
"use_multi"] else layout["showlegend"]
figure = apply_fig_setting(figure,
i,
col,
df,
setting["bar"],
mode="bar",
showlegend=showlegend)
figure = apply_fig_setting(figure,
i,
col,
df,
setting["line"],
mode="lines",
showlegend=showlegend)
figure = apply_fig_setting(figure,
i,
col,
df,
setting["marker"],
mode="markers",
showlegend=showlegend)
xlabel = "xaxis%s" % label_i
layout[xlabel] = {
"range": daterange,
"rangebreaks": rangebreaks,
"scaleanchor": "x1"
}
if not options["use_multi"]:
ylabel = "yaxis%s" % label_i
layout[ylabel] = {"domain": [domain_from, setting["to"]]}
layout[xlabel]["domain"] = [0.0, 1.0]
domain_from = setting["to"]
return figure, layout
def select_weekday(self, date, days):
current = utils.to_datetime(date)
for i in range(days):
current = utils.select_weekday(current -
utils.relativeterm(1, True))
return utils.to_format(current)
import plotly
import plotly.graph_objects as go
from plotly import subplots
from plotly import figure_factory as FF
import dash_utils
sys.path.append("lib")
import utils
import strategy
from loader import Loader, Bitcoin
from simulator import Simulator, SimulatorSetting, SimulatorData
app = dash_utils.create_app()
date = utils.to_format(dt.now())
config = {
"modeBarButtonsToRemove":
["select2d", "lasso2d", "zoomIn2d", "zoomOut2d", "autoScale2d"],
"showAxisDragHandles":
False
}
app.layout = html.Div(
className="container-fluid",
children=[
html.Div(
className="row",
children=[
dcc.Location(id="url", refresh=False),
