def load_bitcoin_ohlc(code, start_date, end_date, with_filter=True, strict=False, time=None): start = utils.to_datetime(start_date) end = utils.to_datetime(end_date) current = start data = None while current <= end: current = utils.to_datetime(utils.to_format(current + relativedelta(days=1), output_format="%Y-%m-%d")) try: data_ = pandas.read_csv(Loader.bitcoin_dir + "/" + str(code) + "/" + utils.to_format(current) + ".csv") except: continue data = data_ if (data is None) else pandas.concat([data, data_]) if data is None: return None data = Loader.format(data, float, date_format="%Y-%m-%d %H:%M:%S") data["low"] = data[["low", "open", "close"]].apply(min, axis=1) data["high"] = data[["high", "open", "close"]].apply(max, axis=1) if with_filter: end_time = "23:59:59" if time is None else utils.format("%s %s" % (end_date, time), output_format="%H:%M:%S") filtered = Loader.filter(data, "%s 00:00:00" % start_date, "%s %s" % (end_date, end_time), strict) if len(filtered) == 0: return None return filtered return data return data
def target_codes(args, terms, strategy_simulator): codes = [] for term in terms: start = utils.to_format(term["start_date"]) end = utils.to_format(term["end_date"]) targets = select_codes(args, start, end, strategy_simulator) codes = list(set(codes + targets)) return codes
def load_portfolio(date, price, length=10, by_day=False): if by_day: d = utils.to_format(utils.select_weekday(date, to_before=False)) else: d = utils.to_format(datetime(date.year, date.month, 1)) try: data = pandas.read_csv("portfolio/financials/%s.csv" % d, header=None) data.columns = ["code", "value", "count", "date"] data = data.iloc[:length] except: data = None return data
def load_portfolio(signals, date, price, length=10, by_day=False): if by_day: d = utils.to_format(utils.select_weekday(date, to_before=False)) else: d = utils.to_format(datetime(date.year, date.month, 1)) try: data = pandas.read_csv("portfolio/%s/%s.csv" % (signals, d)) data = data[data["price"] <= price] data = data.iloc[:length] except: data = None return data
def simulate_params(stocks, terms, strategy_simulator, ignore_manda=True): params = [] strategy_simulator.simulator_setting.strategy = None strategy_creator = strategy_simulator.strategy_creator(stocks["args"]) for term in terms: start = utils.to_format(term["start_date"]) end = utils.to_format(term["end_date"]) codes = select_codes(stocks["args"], start, end, strategy_simulator) select = select_data(codes, stocks, start, end, strategy_creator) params.append((select, utils.to_format(utils.to_datetime(start)), end, ignore_manda)) # print("simulate params:", start, end, utils.timestamp()) return params
def load_tick(code, start_date, end_date, with_filter=True, strict=False, time=None): start = utils.to_datetime(start_date) end = utils.to_datetime(end_date) current = start data = None while current <= end: month = current.strftime("%Y%m") current = utils.to_datetime( utils.to_format(current + relativedelta(months=1), output_format="%Y-%m-01")) try: data_ = pandas.read_csv(Loader.tick_dir + "/" + month + "/" + str(code) + ".csv", header=None) except: continue data = data_ if (data is None) else pandas.concat([data, data_]) if data is None: return None data = Loader.tick_format(data) if with_filter: end_time = "23:59:59" if time is None else utils.format( "%s %s" % (end_date, time), output_format="%H:%M:%S") filtered = Loader.filter(data, "%s 00:00:00" % start_date, "%s %s" % (end_date, end_time), strict) if len(filtered) == 0: return None return filtered return data
def before_ranking(date, ranking_type, before=1): d = utils.to_datetime(date) - utils.relativeterm(before, with_time=True) while not utils.is_weekday(d): d = d - utils.relativeterm(1, with_time=True) d = utils.to_format(d) stocks = Loader.ranking(d, ranking_type) return stocks
def load_portfolio(date, price, length=10, by_day=False): if by_day: d = utils.to_format(utils.select_weekday(date, to_before=False)) else: d = utils.to_format(datetime(date.year, date.month, 1)) try: data = pandas.read_csv("portfolio/high_update/%s.csv" % d, header=None) data.columns = ["code", "price", "count", "date"] data = data[data["price"] <= price] if len(data) < 50 or len(data) > 550: data = None else: data = data.iloc[:length] except: data = None return data
def get_targets(self, args, targets, date, use_cache=False): if args.code is None: if args.instant: date = utils.to_format(utils.select_weekday(utils.to_datetime(date), to_before=False)) targets = list(self.strategy_creator(args, use_cache=use_cache).subject(date)) else: targets = [args.code] return targets
def load_portfolio(date, price, length=10): d = utils.to_format(datetime(date.year, date.month, 1)) try: data = pandas.read_csv("portfolio/financials/%s.csv" % d, header=None) data.columns = ["code", "value", "count", "date"] data = data.iloc[:length] except: data = None return data
def load_portfolio(date, length=10): d = utils.to_format(date) try: data = pandas.read_csv("portfolio/per/%s.csv" % d, header=None) data.columns = ["code", "per", "close"] data = data.iloc[:length] except: data = None return data
def select_codes(self, args, start_date, end_date): codes = [] dates = self.strategy_creator(args, use_cache=True).select_dates(start_date, end_date, instant=args.instant) for d in dates: date = utils.to_format(d) targets = self.get_targets(args, codes, date, use_cache=True) codes = list(set(codes + targets)) return codes
def load_portfolio(date, price, length=10): d = utils.to_format(datetime(date.year, date.month, 1)) try: data = pandas.read_csv("portfolio/new_high/%s.csv" % d, header=None) data.columns = ["code", "price"] data = data[data["price"] <= price] data = data.iloc[:length] except: data = None return data
def load_portfolio(signals, date, price, length=10): # d = utils.to_format(datetime(date.year, date.month, 1)) d = utils.to_format(date) try: data = pandas.read_csv("portfolio/%s/%s.csv" % (signals, d)) data = data[data["price"] <= price] data = data.iloc[:length] except: data = None return data
def load_portfolio(self, date): d = utils.to_format(datetime(date.year, date.month, 1)) try: data = pandas.read_csv("portfolio/high_update/%s.csv" % d, header=None) data.columns = ["code", "price", "count", "date"] data = data[data["price"] <= (self.setting.assets / 250)] data = data.iloc[:10] except: data = None return data
def load_stock(self, start_date, end_date): combination_setting = strategy.CombinationSetting() strategy_creator = CombinationStrategy(combination_setting) codes = [] for date in utils.daterange(utils.to_datetime(start_date), utils.to_datetime(end_date)): codes = list( set(codes + strategy_creator.subject(utils.to_format(date)))) data = None while data is None or len(data) <= self.state_size: self.code = numpy.random.choice(codes) data = Loader.load( self.code, utils.to_format( utils.to_datetime(start_date) - relativedelta(months=12)), end_date) data = utils.add_stats(data) data = Loader.filter(data, start_date, end_date) print("code: [%s]" % self.code) return data
def validation(args, stocks, terms, strategy_simulator, combination_setting, strategy_settings): performances = {} strategy_simulator.strategy_settings = strategy_settings strategy_simulator.combination_setting = combination_setting params = simulate_params(stocks, terms, strategy_simulator, ignore_manda=args.ignore_manda) simulator_setting = copy.deepcopy(strategy_simulator.simulator_setting) if args.verbose or args.apply_compound_interest: print("debug mode") for param in params: _, start_date, end_date, _ = param result = simulate_by_term((strategy_simulator, strategy_settings[-1]) + param) if args.apply_compound_interest: # 複利を適用 strategy_simulator.simulator_setting.assets += result["gain"] performances[utils.to_format( utils.to_datetime_by_term(end_date))] = result else: params = list( map(lambda x: (strategy_simulator, strategy_settings[-1]) + x, params)) results = simulate_by_multiple_term(stocks, params) for result in results: performances[utils.to_format( utils.to_datetime_by_term(result["end_date"]))] = result # 検証スコア score = -evaluator.get_score(args, performances.values(), strategy_simulator.simulator_setting, strategy_settings[-1]) # 結果の表示 ============================================================================= report = create_performance(args, simulator_setting, performances) return score, report, performances
def load_portfolio(self, date, length=10): d = utils.to_format(date) try: data = pandas.read_csv("portfolio/high_update/%s.csv" % d, header=None) data.columns = ["code", "price", "count", "date"] data = data[data["price"] <= (self.setting.assets / 500)] if len(data) < 50 or len(data) > 550: data = None else: data = data.iloc[:length] except: data = None return data
def select_data(codes, stocks, start, end, strategy_creator): select = {"data": {}, "index": stocks["index"], "args": stocks["args"]} args = select["args"] for code in codes: if not code in stocks["data"].keys(): continue start_date = utils.to_format( utils.to_datetime(start) - utils.relativeterm(3)) select["data"][code] = stocks["data"][code].split(start_date, end) select["data"][code] = strategy_creator.add_data( stocks["data"][code], stocks["index"]) return select
def steady_trend_stocks(start_date, end_date, filename="rising_stocks.csv"): current = start_date data = {"all": []} output_format = "%Y%m" while int(utils.format(current, output_format=output_format)) <= int(utils.format(end_date, output_format=output_format)): try: months = utils.format(current, output_format=output_format) d = pandas.read_csv("%s/steady_trend_stocks/%s/%s" % (Loader.settings_dir, months, filename), header=None) d.columns = ['code'] data[months] = d["code"].values.tolist() data["all"] = list(set(data["all"] + data[months])) except: continue finally: current = utils.to_format(utils.to_datetime(current) + utils.relativeterm(1)) return data
def load(args, codes, terms, combination_setting): min_start_date = min(list(map(lambda x: x["start_date"], terms))) prepare_term = utils.relativeterm(args.validate_term) start_date = utils.to_format(min_start_date - prepare_term) end_date = utils.format(args.date) strategy_creator = strategy.load_strategy_creator(args, combination_setting) print("loading %s %s %s %s" % (len(codes), min_start_date, start_date, end_date)) index = strategy.load_index(args, start_date, end_date) data = {} params = list( map( lambda x: { "code": x, "start_date": start_date, "end_date": end_date, "args": args }, codes)) try: p = Pool(args.jobs) ret = p.map(create_simulator_data, params) for r in ret: if r is None: continue data[r.code] = r except KeyboardInterrupt: p.close() exit() finally: p.close() print("loading done") return {"data": data, "index": index, "args": args}
def apply_trace(figure, before, date, code, row, col, rows, cols, layout, options): args = create_args(code) end = date start = utils.to_format( utils.to_datetime(end) - utils.relativeterm(int(before))) simulator_data = strategy.load_simulator_data( args.code, start, end, args, names=["average"] if options["use_multi"] else []) df = simulator_data.middle df = df[df["date"] >= start] df = df.reset_index() # 陽線-> candle.data[1], 陰線 -> candle.data[0] candle = FF.create_candlestick(df["open"], df["high"], df["low"], df["close"], dates=df["date"]) stocks = list(candle.data) figure = add_stats(figure, row, col, stocks, df, []) # ローソク足をグラフに適用 # 日付リスト dates = list(map(lambda x: utils.to_datetime(x), df["date"].astype(str))) rangebreaks = [{ "values": list( filter(lambda x: not utils.is_weekday(x), utils.daterange(dates[0], dates[-1]))) }] # 休日を除外 daterange = [dates[0], dates[-1]] layout_settings = create_layout_settings(options) if options["use_multi"]: setting = layout_settings[0] setting["line"] = [setting["line"][0]] # candle with average layout_settings = [setting] # =================================================================== # layout_settingをもとに各グラフにデータを入れる domain_from = 0 for setting in sorted(layout_settings, key=lambda x: x["to"]): i = row if options["use_multi"] else setting["id"] label_i = ( (col - 1) + (row - 1) * cols) + 1 if options["use_multi"] else setting["id"] # multiの場合同じlegendがいくつも並ぶので最初のだけ有効に showlegend = layout["showlegend"] and row == 1 and col == 1 if options[ "use_multi"] else layout["showlegend"] figure = apply_fig_setting(figure, i, col, df, setting["bar"], mode="bar", showlegend=showlegend) figure = apply_fig_setting(figure, i, col, df, setting["line"], mode="lines", showlegend=showlegend) figure = apply_fig_setting(figure, i, col, df, setting["marker"], mode="markers", showlegend=showlegend) xlabel = "xaxis%s" % label_i layout[xlabel] = { "range": daterange, "rangebreaks": rangebreaks, "scaleanchor": "x1" } if not options["use_multi"]: ylabel = "yaxis%s" % label_i layout[ylabel] = {"domain": [domain_from, setting["to"]]} layout[xlabel]["domain"] = [0.0, 1.0] domain_from = setting["to"] return figure, layout
def select_weekday(self, date, days): current = utils.to_datetime(date) for i in range(days): current = utils.select_weekday(current - utils.relativeterm(1, True)) return utils.to_format(current)
import plotly import plotly.graph_objects as go from plotly import subplots from plotly import figure_factory as FF import dash_utils sys.path.append("lib") import utils import strategy from loader import Loader, Bitcoin from simulator import Simulator, SimulatorSetting, SimulatorData app = dash_utils.create_app() date = utils.to_format(dt.now()) config = { "modeBarButtonsToRemove": ["select2d", "lasso2d", "zoomIn2d", "zoomOut2d", "autoScale2d"], "showAxisDragHandles": False } app.layout = html.Div( className="container-fluid", children=[ html.Div( className="row", children=[ dcc.Location(id="url", refresh=False),