class StockService(BaseService): def __init__(self) -> None: super().__init__() self.__calc_service = CalcService() self.__crawl_client: CrawlClient = CrawlClient() self.__email_client: EmailClient = EmailClient() def get_symbol(self, symbol: str, instrument: str) -> SM: if StringUtils.isNullOrWhitespace(symbol): return None symbol = symbol.strip().upper() return BaseService._get_first( SM, [SM.symbol == symbol, SM.instrument == instrument]) def get_symbols(self, instrument: str = '', exclude_status: List[int] = []) -> List[SM]: query = db.session.query(SM) if instrument != '': query = query.filter(SM.instrument == instrument) for status in exclude_status: query = query.filter(SM.status.op('|')(status) != SM.status) query = query.order_by(SM.symbol) return query.all() def update_symbol(self, model: SM) -> int: if not model: raise BadRequestException() org_symbol: SM = self.get_symbol(model.symbol, model.instrument) if not org_symbol: raise NotFoundException(model.__class__.__name__, 'symbol', model.symbol) org_symbol.name = model.name org_symbol.status = model.status BaseService._update() return 1 def delete_old_prices(self) -> int: error: Exception = None try: thirty_years_ago: datetime = datetime.now() - timedelta(days=365 * 30) LogUtils.debug( 'Deleting stock price older than {0}'.format(thirty_years_ago)) db.session.query(SPD).filter( SPD.price_date <= thirty_years_ago).delete() db.session.commit() except Exception as ex: error = ex finally: self.__email_client.send_html( subject=AppConsts.EMAIL_SUBJECT_DELETE_PRICES, template_path=AppConsts.TEMPLATE_PATH_DELETE_PRICES, model={'errors': [error] if error else []}) if error: LogUtils.error('Delete Price Error', error) return 1 def get_single_stock_price_daily(self, symbol_id: int, price_date: date) -> SPD: return BaseService._get_first( SPD, [SPD.symbol_id == symbol_id, SPD.price_date == price_date]) def get_last_single_stock_price_daily(self, symbol_id: int) -> SPD: return db.session.query(SPD).filter( SPD.symbol_id == symbol_id).order_by( SPD.price_date.desc()).first() def get_single_etf_price_daily(self, symbol_id: int, price_date: date) -> SPD: return BaseService._get_first( EPD, [EPD.symbol_id == symbol_id, EPD.price_date == price_date]) def get_vw_symbol_spd_as_df(self, symbol_id: int = None, symbol_ids: List = None, date_from: date = None, date_to: date = None) -> DataFrame: query = db.session.query(VSPD) if symbol_id and symbol_id > 0: query = query.filter(VSPD.symbol_id == symbol_id) if symbol_ids and len(symbol_ids) > 0: query = query.filter(VSPD.symbol_id.in_(symbol_ids)) if date_from: query = query.filter(VSPD.price_date >= date_from) if date_to: query = query.filter(VSPD.price_date <= date_to) return pd.read_sql(sql=query.statement, con=db.session.bind, index_col=[ AppConsts.PRICE_COL_SYMBOL_ID, AppConsts.PRICE_COL_DATE ]) def get_vw_symbol_epd_as_df(self, symbol_id: int = None, symbol_ids: List = None, date_from: date = None, date_to: date = None) -> DataFrame: query = db.session.query(VEPD) if symbol_id and symbol_id > 0: query = query.filter(VEPD.symbol_id == symbol_id) if symbol_ids and len(symbol_ids) > 0: query = query.filter(VEPD.symbol_id.in_(symbol_ids)) if date_from: query = query.filter(VEPD.price_date >= date_from) if date_to: query = query.filter(VEPD.price_date <= date_to) return pd.read_sql(sql=query.statement, con=db.session.bind, index_col=[ AppConsts.PRICE_COL_SYMBOL_ID, AppConsts.PRICE_COL_DATE ]) def get_stock_prices_daily(self, symbol_id: int, date_from: date = None, date_to: date = None) -> List[SPD]: query = db.session.query(SPD) if symbol_id > 0: query = query.filter(SPD.symbol_id == symbol_id) if date_from: query = query.filter(SPD.price_date >= date_from) if date_to: query = query.filter(SPD.price_date <= date_to) return query.order_by(SPD.price_date).all() def get_etf_prices_daily(self, symbol_id: int, date_from: date = None, date_to: date = None) -> List[EPD]: query = db.session.query(EPD) if symbol_id > 0: query = query.filter(EPD.symbol_id == symbol_id) if date_from: query = query.filter(EPD.price_date >= date_from) if date_to: query = query.filter(EPD.price_date <= date_to) return query.order_by(EPD.price_date).all() def get_financial(self, symbol_id: int, year: int, quarter: int) -> FN: return BaseService._get_first(FN, [ FN.symbol_id == symbol_id, FN.year == year, FN.quarter == quarter ]) def get_price_dataframe(self, prices: List[Any]) -> DataFrame: if not prices \ or not isinstance(prices, List) \ or not isinstance(prices[0], (SPD, EPD)): return None df: DataFrame = pd.DataFrame( data=[ [ p.id, p.symbol_id, # idx = 0, 0 p.price_date, # idx = 0, 1 NumberUtils.to_float(p.open_price), NumberUtils.to_float(p.high_price), NumberUtils.to_float(p.low_price), NumberUtils.to_float(p.close_price), p.volume ] for p in prices ], columns=AppConsts.PRICE_COLS) df = df.set_index( [AppConsts.PRICE_COL_SYMBOL_ID, AppConsts.PRICE_COL_DATE]) return df def get_strategy_service(self, strategy_type: str, strategy_request: Any, symbols: List[SM], prices: DataFrame) -> BaseStrategyService: if strategy_type == AppConsts.STRATEGY_DEMO: return DemoStrategyService(strategy_request, symbols, prices) if strategy_type == AppConsts.STRATEGY_ABZ: return AbzStrategyService(strategy_request, symbols, prices) if strategy_type == AppConsts.STRATEGY_DBB: return DoubleBollingerBandsStrategyService(strategy_request, symbols, prices) if strategy_type == AppConsts.STRATEGY_DOUBLE_BOTTOMS: return DoubleBottomsStrategyService(strategy_request, symbols, prices) if strategy_type == AppConsts.STRATEGY_DOUBLE_TOPS: return DoubleTopsStrategyService(strategy_request, symbols, prices) if strategy_type == AppConsts.STRATEGY_INVERTED_HEAD_AND_SHOULDERS: return InvertedHeadAndShouldersStrategyService( strategy_request, symbols, prices) if strategy_type == AppConsts.STRATEGY_SMA_CROSSOVER: return SmaCrossoverStrategyService(strategy_request, symbols, prices) if strategy_type == AppConsts.STRATEGY_TURNAROUND_TUESDAY: return TurnaroundTuesdayStrategyService(strategy_request, symbols, prices) if strategy_type == AppConsts.STRATEGY_TEMA_AND_VWMA: return TEMAandVWMAStrategyService(strategy_request, symbols, prices) return None def get_dates(self, prices: DataFrame, start_date: date, end_date: date) -> DataFrame: if prices.empty \ or not start_date \ or not end_date \ or start_date > end_date \ or not AppConsts.PRICE_COL_DATE in prices.index.names: return None dates: DataFrame = pd.DataFrame( prices.index.get_level_values(AppConsts.PRICE_COL_DATE).unique()) dates = dates.loc[(dates[AppConsts.PRICE_COL_DATE] >= start_date) & (dates[AppConsts.PRICE_COL_DATE] <= end_date)] dates[AppConsts.CUSTOM_COL_PREV_DATE] = dates[ AppConsts.PRICE_COL_DATE].shift(1) dates[AppConsts.CUSTOM_COL_NEXT_DATE] = dates[ AppConsts.PRICE_COL_DATE].shift(-1) dates[AppConsts.CUSTOM_COL_NEXT_NEXT_DATE] = dates[ AppConsts.PRICE_COL_DATE].shift(-2) return dates def get_no_of_shares(self, capital: float, pct_risk_per_trade: float, volume_limit: float, price: Series, slippage: int = None, is_slip_up: bool = True) -> int: if not AppConsts.PRICE_COL_OPEN in price.index \ or not AppConsts.CUSTOM_COL_ADV in price.index: return 0 open_price: float = price.loc[AppConsts.PRICE_COL_OPEN] if slippage: if is_slip_up: open_price = NumberUtils.round(open_price + ( open_price * AppConsts.BASIS_POINT * slippage)) else: open_price = NumberUtils.round(open_price - ( open_price * AppConsts.BASIS_POINT * slippage)) no_of_shares: int = NumberUtils.to_floor(capital * pct_risk_per_trade / 100 / open_price) adv: float = price.loc[AppConsts.CUSTOM_COL_ADV] if price.loc[ AppConsts.CUSTOM_COL_ADV] > 0 else price.loc[ AppConsts.PRICE_COL_VOLUME] max_volume: float = NumberUtils.to_int(adv * volume_limit / 100) if no_of_shares > max_volume: LogUtils.warning( 'Capping max_volume adv={0}, no_of_shares={1}, max_volume={2}'. format(adv, no_of_shares, max_volume)) no_of_shares = max_volume return no_of_shares def get_sample_prices_for_charts(self, req: CR) -> List[List[SPC]]: LogUtils.debug('START') ret: List[List[SPC]] = [] if not req or not req.is_valid_model(): raise BadRequestException() # region Init Symbols symbols: List[SymbolMaster] = self.get_symbols( AppConsts.INSTRUMENT_STOCK) if req.is_random_symbols: shuffle(symbols) symbols = symbols[:req.no_of_charts * 5] else: symbols = [s for s in symbols if s.symbol == req.symbol.upper()] if not symbols: return ret symbol_ids: List[int] = [symbol.id for symbol in symbols] # endregion LogUtils.debug('Symbols count={0}'.format(len(symbol_ids))) # region Init Prices prices = self.get_vw_symbol_spd_as_df( symbol_ids=symbol_ids, date_from=(req.date_from_obj - timedelta(days=300)), # for sma (will filter later) date_to=req.date_to_obj) # endregion LogUtils.debug('Prices Init Shape={0}'.format(prices.shape)) prices[AppConsts.CUSTOM_COL_PV] = prices[ AppConsts.PRICE_COL_CLOSE] * prices[AppConsts.PRICE_COL_VOLUME] prices[AppConsts.CUSTOM_COL_ADPV] = 0 for symbol in symbols: symbol_prices: DataFrame = prices.loc[[symbol.id]] if symbol_prices.empty: continue # region ADPV (50) adpvs: Series = symbol_prices[AppConsts.CUSTOM_COL_PV].rolling( window=50).mean() prices[AppConsts.CUSTOM_COL_ADPV].update(adpvs) adpv: float = adpvs.tail(1)[0] if adpv < AppConsts.ADPV_MIN_DFLT: continue symbol_prices = prices.loc[[symbol.id]] # endregion LogUtils.debug('START-Symbol={0},ADPV={1}'.format( symbol.symbol, adpv)) # SMA prices = self.__calc_service.append_sma( prices=prices, index=[symbol.id], sma_period=req.sma_period_1, sma_column_name=AppConsts.CUSTOM_COL_SMA_PERIOD_1) prices = self.__calc_service.append_sma( prices=prices, index=[symbol.id], sma_period=req.sma_period_2, sma_column_name=AppConsts.CUSTOM_COL_SMA_PERIOD_2) # Exponential Price Smoothing prices = self.__calc_service.append_exponential_smoothing_price( prices=prices, index=[symbol.id], alpha=req.exponential_smoothing_alpha) # Exponential Price Smoothing Max/Min prices = self.__calc_service.append_exponential_smoothing_max_min( prices=prices, index=[symbol.id], exponential_smoothing_max_min_diff=req. exponential_smoothing_max_min_diff) # Double Bottom prices = self.__calc_service.append_double_bottoms( prices=prices, index=[symbol.id], price_column=AppConsts.PRICE_COL_CLOSE, smooth_price_column=AppConsts. CUSTOM_COL_EXPONENTIAL_SMOOTHING_PRICE, max_column=AppConsts.CUSTOM_COL_EXPONENTIAL_SMOOTHING_MAX, min_column=AppConsts.CUSTOM_COL_EXPONENTIAL_SMOOTHING_MIN, double_bottoms_diff=5) # ABZ prices = self.__calc_service.append_abz( prices=prices, index=[symbol.id], abz_er_period=req.abz_er_period, abz_std_distance=req.abz_std_distance, abz_constant_k=req.abz_constant_k) symbol_prices = prices.loc[[symbol.id]] ret.append([ SPC(i, row) for i, row in symbol_prices[symbol_prices.index.get_level_values( AppConsts.PRICE_COL_DATE) >= req.date_from_obj].iterrows() ]) if len(ret) == req.no_of_charts: return ret LogUtils.debug('END') return ret def get_sp500_mismatches(self, is_missing: bool) -> None: ret: List[SM] = [] db_symbol_masters: List[SM] = self.get_symbols( AppConsts.INSTRUMENT_STOCK, [AppConsts.SYMBOL_STATUS_ARCHIVED]) if not db_symbol_masters: return ret sp500_df: DataFrame = self.__crawl_client.get_html_table( AppConsts.WIKI_SP500_URL, AppConsts.WIKI_SP500_ELEMENT_ID, [AppConsts.WIKI_SP500_COL_SYMBOL]) db_symbols: List[str] = [s.symbol for s in db_symbol_masters] if is_missing: # Get SP500 symbols not in db for idx, row in sp500_df.iterrows(): if idx and not idx.strip().upper() in db_symbols: tmp: SM = SM() tmp.symbol = idx.strip().upper() tmp.name = row.loc[AppConsts.WIKI_SP500_COL_SYMBOL_NAME] ret.append(tmp) else: # Get symbols in db that is not SP500 for symbol in db_symbol_masters: if not symbol.symbol in sp500_df.index: ret.append(symbol) return ret
class BackTestService(BaseService): def __init__(self) -> None: super().__init__() self.__stock_service = StockService() self.__calc_service = CalcService() def run(self, req: BackTestRunRequest) -> BackTestRunResponse: if not req or not req.is_valid_model(): raise BadRequestException() response: BackTestRunResponse = BackTestRunResponse(req) # Init Symbols symbols: List[SymbolMaster] = self.__get_symbols__(req) # Init Prices prices: DataFrame = self.__get_prices__(req, symbols) # Do Base Preparation prices[AppConsts.CUSTOM_COL_PV] = prices[ AppConsts.PRICE_COL_CLOSE] * prices[AppConsts.PRICE_COL_VOLUME] for s in symbols: prices = self.__calc_service.append_sma( prices=prices, index=[s.id], sma_period=AppConsts.ADV_PERIOD_DFLT, sma_column_name=AppConsts.CUSTOM_COL_ADV, target_column=AppConsts.PRICE_COL_VOLUME) prices = self.__calc_service.append_sma( prices=prices, index=[s.id], sma_period=AppConsts.ADPV_PERIOD_DFLT, sma_column_name=AppConsts.CUSTOM_COL_ADPV, target_column=AppConsts.CUSTOM_COL_PV) LogUtils.debug('Prices shape after base preparation={0}'.format( prices.shape)) # region Init Service strategy_service: Any = self.__stock_service.get_strategy_service( req.strategy_type, req.strategy_request, symbols, prices) if not strategy_service or not strategy_service._is_valid_request(): raise BadRequestException() strategy_service._do_preparations() LogUtils.debug('Prices shape after strategy preparation={0}'.format( prices.shape)) # endregion LogUtils.debug(prices.info()) # region Init Dates start_date: date = DateUtils.add_business_days(req.date_from_obj, -1) start_date = DateUtils.add_business_days(start_date, 1) start_date_str: str = DateUtils.to_string(start_date) end_date: date = DateUtils.add_business_days(req.date_to_obj, -1) dates: DataFrame = self.__stock_service.get_dates( prices, start_date, end_date) LogUtils.debug( 'Dates actual_start={0}, actual_end={1}, shape={2}'.format( start_date, end_date, dates.shape)) # endregion # region Loop Dates strategy_item: BackTestResultItem = next( b for b in response.back_test_result_items if b.target == req.strategy_type) strategy_item.capital[start_date_str] = req.start_capital strategy_item.capital_available[start_date_str] = req.start_capital portfolio: Dict = {} for i, date_row in dates.iterrows(): current_date = date_row[AppConsts.PRICE_COL_DATE] current_date_str: str = DateUtils.to_string(current_date) next_date = date_row[AppConsts.CUSTOM_COL_NEXT_DATE] next_date_str = DateUtils.to_string(next_date) next_next_date = date_row[AppConsts.CUSTOM_COL_NEXT_NEXT_DATE] shuffle(symbols) for symbol in symbols: has_price: bool = (symbol.id, current_date) in prices.index if not has_price: continue price: Series = prices.loc[symbol.id, current_date] if symbol.instrument == AppConsts.INSTRUMENT_ETF: # region Benchmark b_result_item: BackTestResultItem = next( b for b in response.back_test_result_items if b.target == symbol.symbol) if not b_result_item: continue if not b_result_item.transactions: no_of_shares: int = self.__stock_service.get_no_of_shares( req.start_capital, req.pct_risk_per_trade, req.volume_limit, price, req.slippage) if no_of_shares == 0: LogUtils.warning('0 shares for ETF={0}'.format( symbol.symbol)) continue b_transaction: Transaction = Transaction() b_transaction.symbol_master = symbol b_transaction.action = AppConsts.ACTION_BUY b_transaction.start_date = current_date b_transaction.start_price = price.loc[ AppConsts.PRICE_COL_OPEN] b_transaction.no_of_shares = no_of_shares b_result_item.transactions.append(b_transaction) b_result_item.capital[ current_date_str] = req.start_capital else: b_transaction: Transaction = b_result_item.transactions[ 0] b_transaction.end_date = current_date b_transaction.end_price = price.loc[ AppConsts.PRICE_COL_CLOSE] b_transaction.set_readonly_props() b_result_item.capital[ current_date_str] = self.__calc_benchmark_capital( req, b_transaction.start_price, b_transaction.end_price, b_transaction.no_of_shares) b_result_item.ttl_no_days += 1 # endregion else: # region Strategy strategy_service._do_calculations(symbol.id, current_date) action: str = strategy_service._get_action() is_in_position: bool = symbol.id in portfolio if not is_in_position: if len(portfolio ) == req.portfolio_max: # todo: prioritize? continue if current_date == end_date or next_date >= end_date: continue has_next_price: bool = (symbol.id, next_date) in prices.index has_next_next_price: bool = ( symbol.id, next_next_date) in prices.index if not has_next_price or not has_next_next_price: continue adv: float = price.loc[ AppConsts.CUSTOM_COL_ADV] if price.loc[ AppConsts.CUSTOM_COL_ADV] > 0 else price.loc[ AppConsts.PRICE_COL_VOLUME] if adv < req.adv_min: continue adpv: float = price.loc[ AppConsts.CUSTOM_COL_ADPV] if price.loc[ AppConsts.CUSTOM_COL_ADPV] > 0 else price.loc[ AppConsts.CUSTOM_COL_PV] if adpv < req.adpv_min: continue next_price: Series = prices.loc[symbol.id, next_date] has_entry_conditions: bool = strategy_service._has_entry_conditions( symbol.id, current_date) if has_entry_conditions: no_of_shares: int = self.__stock_service.get_no_of_shares( strategy_item. capital_available[current_date_str], req.pct_risk_per_trade, req.volume_limit, next_price, req.slippage, action == AppConsts.ACTION_BUY) if no_of_shares == 0: continue trans: Transaction = Transaction() trans.symbol_master = symbol trans.action = action trans.start_date = next_date trans.start_price = next_price.loc[ AppConsts.PRICE_COL_OPEN] trans.no_of_shares = no_of_shares trans_amount: float = NumberUtils.round( trans.start_price * no_of_shares) strategy_item.capital_available[ current_date_str] -= trans_amount # Add to portfolio portfolio[symbol.id] = trans elif is_in_position: has_exit_conditions: bool = strategy_service._has_exit_conditions( symbol.id, current_date) has_next_next_price: bool = ( symbol.id, next_next_date) in prices.index if next_date == end_date or not has_next_next_price or has_exit_conditions: next_price: Series = prices.loc[symbol.id, next_date] next_open_price: float = next_price.loc[ AppConsts.PRICE_COL_OPEN] slippage_price: float = 0 if action == AppConsts.ACTION_BUY: slippage_price: float = NumberUtils.round( next_open_price - (next_open_price * AppConsts.BASIS_POINT * req.slippage)) else: slippage_price: float = NumberUtils.round( next_open_price + (next_open_price * AppConsts.BASIS_POINT * req.slippage)) trans: Transaction = portfolio.get(symbol.id) trans.end_date = next_date trans.end_price = slippage_price trans.set_readonly_props() strategy_item.transactions.append(trans) if action == AppConsts.ACTION_BUY: trans_amount = NumberUtils.round( trans.end_price * trans.no_of_shares) strategy_item.capital_available[ current_date_str] += trans_amount else: init_trans_amount = NumberUtils.round( trans.start_price * trans.no_of_shares) strategy_item.capital_available[ current_date_str] += init_trans_amount strategy_item.capital_available[ current_date_str] += trans.change_in_capital # Remove from portfolio portfolio.pop(symbol.id, None) # endregion # capital = capital available + capital in portfolio capital: float = strategy_item.capital_available[current_date_str] for key, val in portfolio.items(): price: Series = prices.loc[key, current_date] capital += price.loc[ AppConsts.PRICE_COL_CLOSE] * val.no_of_shares strategy_item.capital[current_date_str] = NumberUtils.round( capital) strategy_item.ttl_no_days += 1 strategy_item.capital[next_date_str] = strategy_item.capital[ current_date_str] strategy_item.capital_available[ next_date_str] = strategy_item.capital_available[ current_date_str] # endregion for result_item in response.back_test_result_items: result_item.set_readonly_props() return response def __get_symbols__(self, req: BackTestRunRequest) -> List[SymbolMaster]: if not req or req.test_limit_symbol <= 0: raise BadRequestException() symbols: List[SymbolMaster] = self.__stock_service.get_symbols( AppConsts.INSTRUMENT_STOCK, [AppConsts.SYMBOL_STATUS_EXCLUDE_TRADE]) if not symbols: raise DbConnectionException() shuffle(symbols) symbols = symbols[:req.test_limit_symbol] if req.benchmark_etfs: for benchmark_etf in req.benchmark_etfs: etf_symbol: SymbolMaster = self.__stock_service.get_symbol( benchmark_etf, AppConsts.INSTRUMENT_ETF) if not etf_symbol: continue symbols.append(etf_symbol) LogUtils.debug('Symbol Count={0}'.format(len(symbols))) return symbols def __get_prices__(self, req: BackTestRunRequest, symbols: List[SymbolMaster]) -> DataFrame: if not req or not symbols: raise BadRequestException() price_items: List[Any] = [] for symbol in symbols: temp: List[Any] = [] if symbol.instrument == AppConsts.INSTRUMENT_STOCK: temp = self.__stock_service.get_stock_prices_daily( symbol.id, req.date_from_obj, req.date_to_obj) elif symbol.instrument == AppConsts.INSTRUMENT_ETF: temp = self.__stock_service.get_etf_prices_daily( symbol.id, req.date_from_obj, req.date_to_obj) if temp: price_items.extend(temp) prices: DataFrame = self.__stock_service.get_price_dataframe( price_items) LogUtils.debug('Prices Init Shape={0}'.format(prices.shape)) return prices def __calc_benchmark_capital(self, req: BackTestRunRequest, start_price: float, end_price: float, no_of_shares: int) -> float: capital: float = req.start_capital - (start_price * no_of_shares) return NumberUtils.round(capital + (end_price * no_of_shares))