def render_to_response(self, context): qs = context['paginator'].object_list.all( ) # all() to get a fresh queryset instance qs = list(qs.values_list('asx_code', flat=True)) if len(qs) == 0: warning(self.request, "No stocks to report") sentiment_data, df, top10, bottom10, n_stocks = (None, None, None, None, 0) else: sentiment_data, df, top10, bottom10, n_stocks = plot_heatmap( qs, n_top_bottom=self.n_top_bottom) context.update({ 'most_recent_date': self.as_at_date, 'sentiment_heatmap': sentiment_data, 'watched': user_watchlist( self.request.user), # to ensure bookmarks are correct 'n_top_bottom': self.n_top_bottom, 'best_ten': top10, 'worst_ten': bottom10, 'title': 'Find by dividend yield or P/E', 'sentiment_heatmap_title': "Recent sentiment: {} total stocks".format(n_stocks), }) add_messages(self.request, context) return super().render_to_response(context)
def get_queryset(self, **kwargs): # user never run this view before? if kwargs == {}: print("WARNING: no form parameters specified - returning empty queryset") return Quotation.objects.none() self.sector = kwargs.get("sector", self.sector) self.sector_id = int(Sector.objects.get(sector_name=self.sector).sector_id) wanted_stocks = all_sector_stocks(self.sector) print("Found {} stocks matching sector={}".format(len(wanted_stocks), self.sector)) mrd = latest_quotation_date('ANZ') report_top_n = kwargs.get('report_top_n', None) report_bottom_n = kwargs.get('report_bottom_n', None) if report_top_n is not None or report_bottom_n is not None: cip_sum = selected_cached_stocks_cip(wanted_stocks, Timeframe(past_n_days=90)).transpose().sum().to_frame(name="percent_cip") #print(cip_sum) top_N = set(cip_sum.nlargest(report_top_n, "percent_cip").index) if report_top_n is not None else set() bottom_N = set(cip_sum.nsmallest(report_bottom_n, "percent_cip").index) if report_bottom_n is not None else set() wanted_stocks = top_N.union(bottom_N) print("Requesting valid quotes for {} stocks".format(len(wanted_stocks))) self.qs = valid_quotes_only(mrd).filter(asx_code__in=wanted_stocks) if len(self.qs) < len(wanted_stocks): got = set([q.asx_code for q in self.qs.all()]) missing_stocks = wanted_stocks.difference(got) warning(self.request, f"could not obtain quotes for all stocks as at {mrd}: {missing_stocks}") return self.qs
def market_cap_data_factory(ld: LazyDictionary) -> pd.DataFrame: dates = ld["sector_timeframe"].all_dates() # print(dates) assert len(dates) > 90 result_df = None adjusted_dates = [] for the_date in [dates[0], dates[-1], dates[-30], dates[-90]]: print(f"Before valid_quotes_only for {the_date}") quotes, actual_trading_date = valid_quotes_only( the_date, ensure_date_has_data=True) print(f"After valid_quotes_only for {the_date}") print(f"Before make quotes {actual_trading_date}") print(len(quotes)) df = make_quote_df(quotes, ld["asx_codes"], actual_trading_date) print("After make_quote_df") result_df = df if result_df is None else result_df.append(df) if the_date != actual_trading_date: adjusted_dates.append(the_date) if len(adjusted_dates) > 0: warning( request, "Some dates were not trading days, adjusted: {}".format( adjusted_dates), ) return result_df
def rule_at_end_of_daily_range(state: dict): """ Award 1 point if the price at the end of the day is within 20% of the daily trading range (either end) Otherwise 0. """ assert state is not None day_low_high_df = state.get('day_low_high_df') date = state.get('date') threshold = state.get('daily_range_threshold') try: day_low = day_low_high_df.at[date, 'day_low_price'] day_high = day_low_high_df.at[date, 'day_high_price'] last_price = day_low_high_df.at[date, 'last_price'] if np.isnan(day_low) and np.isnan(day_high): return 0 day_range = (day_high - day_low) * threshold # 20% at either end of daily range if last_price >= day_high - day_range: return 1 elif last_price <= day_low + day_range: return -1 # else FALLTHRU... except KeyError: stock = state.get('stock') warning( None, "Unable to obtain day low/high and last_price for {} on {}".format( stock, date)) return 0
def recalc_queryset(self, **kwargs): if kwargs == {} or not any( ["name" in kwargs, "activity" in kwargs, "sector" in kwargs]): return Quotation.objects.none() wanted_name = kwargs.get("name", "") wanted_activity = kwargs.get("activity", "") if len(wanted_name) > 0 or len(wanted_activity) > 0: matching_companies = find_named_companies(wanted_name, wanted_activity) else: matching_companies = all_stocks() sector = kwargs.get("sector", self.DEFAULT_SECTOR) sector_id = int(Sector.objects.get(sector_name=sector).sector_id) sector_stocks = all_sector_stocks(sector) if kwargs.get("sector_enabled", False): matching_companies = matching_companies.intersection(sector_stocks) print("Found {} companies matching: name={} or activity={}".format( len(matching_companies), wanted_name, wanted_activity)) report_top_n = kwargs.get("report_top_n", None) report_bottom_n = kwargs.get("report_bottom_n", None) self.timeframe = Timeframe(past_n_days=90) ld = LazyDictionary() ld["sector"] = sector ld["sector_id"] = sector_id ld["sector_companies"] = sector_stocks if len(matching_companies) > 0: ld["cip_df"] = selected_cached_stocks_cip(matching_companies, self.timeframe) else: ld["cip_df"] = pd.DataFrame() ld["sector_performance_df"] = lambda ld: make_sector_performance_dataframe( ld["cip_df"], ld["sector_companies"]) ld["sector_performance_plot"] = lambda ld: self.sector_performance(ld) self.ld = ld wanted_stocks = self.filter_top_bottom(ld, matching_companies, report_top_n, report_bottom_n) print("Requesting valid quotes for {} stocks".format( len(wanted_stocks))) quotations_as_at, actual_mrd = valid_quotes_only( "latest", ensure_date_has_data=True) ret = quotations_as_at.filter(asx_code__in=wanted_stocks) if len(ret) < len(wanted_stocks): got = set([q.asx_code for q in self.qs.all()]) if self.qs else set() missing_stocks = wanted_stocks.difference(got) warning( self.request, f"could not obtain quotes for all stocks as at {actual_mrd}: {missing_stocks}", ) print("Showing results for {} companies".format( len(matching_companies))) ret, _ = latest_quote(tuple(matching_companies)) return ret
def detect_outliers(stocks: list, all_stocks_cip: pd.DataFrame, rules=None): """ Returns a dataframe describing those outliers present in stocks based on the provided rules. """ if rules is None: rules = default_point_score_rules() str_rules = { str(r):r for r in rules } rows = [] stocks_by_sector_df = stocks_by_sector() # NB: ETFs in watchlist will have no sector stocks_by_sector_df.index = stocks_by_sector_df['asx_code'] for stock in stocks: #print("Processing stock: ", stock) try: sector = stocks_by_sector_df.at[stock, 'sector_name'] sector_companies = list(stocks_by_sector_df.loc[stocks_by_sector_df['sector_name'] == sector].asx_code) # day_low_high() may raise KeyError when data is currently being fetched, so it appears here... day_low_high_df = day_low_high(stock, all_stocks_cip.columns) except KeyError: warning(None, "Unable to locate watchlist entry: {} - continuing without it".format(stock)) continue state = { 'day_low_high_df': day_low_high_df, # never changes each day, so we init it here 'all_stocks_change_in_percent_df': all_stocks_cip, 'stock': stock, 'daily_range_threshold': 0.20, # 20% at either end of the daily range gets a point } points_by_rule = defaultdict(int) for date in all_stocks_cip.columns: market_avg = all_stocks_cip[date].mean() sector_avg = all_stocks_cip[date].filter(items=sector_companies).mean() stock_move = all_stocks_cip.at[stock, date] state.update({ 'market_avg': market_avg, 'sector_avg': sector_avg, 'stock_move': stock_move, 'date': date }) for rule_name, rule in str_rules.items(): points_by_rule[rule_name] += rule(state) d = { 'stock': stock } d.update(points_by_rule) rows.append(d) df = pd.DataFrame.from_records(rows) df = df.set_index('stock') print(df) from pyod.models.iforest import IForest clf = IForest() clf.fit(df) scores = clf.predict(df) results = [row[0] for row, value in zip(df.iterrows(), scores) if value > 0] #print(results) print("Found {} outlier stocks".format(len(results))) return results
def render_to_response(self, context): context.update({ "title": "Find companies by financial metric", "sentiment_heatmap_title": "Matching stock sentiment", }) warning( self.request, "Due to experimental data ingest, results may be wrong/inaccurate/misleading. Use at own risk", ) return show_companies( self.object_list, # ie. return result from self.get_queryset() self.request, Timeframe(past_n_days=30), context, self.template_name, )
def form_valid(self, form): exclude = form.cleaned_data["excluded_stocks"] n_days = form.cleaned_data["n_days"] algo = form.cleaned_data["method"] portfolio_cost = form.cleaned_data["portfolio_cost"] exclude_price = form.cleaned_data.get("exclude_price", None) max_stocks = form.cleaned_data.get("max_stocks", 80) stocks = self.stocks() if exclude is not None: if isinstance(exclude, str): exclude = exclude.split(",") stocks = set(stocks).difference(exclude) if form.cleaned_data["exclude_etfs"]: stocks = set(stocks).difference(all_etfs()) print(f"After excluding ETFs: {len(stocks)} stocks remain") self.timeframe = Timeframe(past_n_days=n_days) self.results = optimise_portfolio( stocks, self.timeframe, algo=algo, max_stocks=max_stocks, total_portfolio_value=portfolio_cost, exclude_price=exclude_price, warning_cb=lambda msg: warning(self.request, msg), returns_by=form.cleaned_data.get("returns_by", None), ) return render(self.request, self.template_name, self.get_context_data())
def form_valid(self, form): df = fetch_dataframe(self.data_flow.name) if df is None or len(df) == 0: raise Http404(f"Unable to load dataframe: {self.data_flow}") filter_performance = [] for k, v in form.cleaned_data.items(): rows_at_start = len(df) print( f"Filtering rows for {k}: total {rows_at_start} rows at start") k = k[len("dimension_"):] if rows_at_start < 10000: unique_values_left = df[k].unique() else: unique_values_left = set() df = df[df[k] == v] rows_at_end = len(df) filter_performance.append( (k, v, rows_at_start, rows_at_end, unique_values_left)) print(f"After filtering: now {rows_at_end} rows") if len(df) == 0: warning(self.request, f"No rows of data left after filtering: {k} {v}") break plot = None plot_title = "" if len(df) > 0: plot_title, x_axis_column, y_axis_column, df = detect_dataframe( df, self.data_flow) plot = (p9.ggplot(df, p9.aes(x=x_axis_column, y=y_axis_column)) + p9.geom_point() + p9.geom_line()) plot = user_theme(plot) context = self.get_context_data() cache_key = "-".join(sorted(form.cleaned_data.values())) + "-ecb-plot" context.update({ "dataflow": self.data_flow, "dataflow_name": self.data_flow.name, "filter_performance": filter_performance, "plot_title": plot_title, "plot_uri": cache_plot(cache_key, lambda: plot), }) return render(self.request, self.template_name, context)
def get_initial(self, **kwargs): stock = kwargs.get("stock", self.kwargs.get("stock")) amount = kwargs.get("amount", self.kwargs.get("amount", 5000.0)) user = self.request.user validate_stock(stock) validate_user(user) quote, latest_date = latest_quote(stock) cur_price = quote.last_price if cur_price >= 1e-6: return { "asx_code": stock, "user": user, "buy_date": latest_date, "price_at_buy_date": cur_price, "amount": amount, "n": int(amount / cur_price), } else: warning(self.request, "Cannot buy {} as its price is zero/unknown".format(stock)) return {}
def get_initial(self, **kwargs): stock = self.kwargs.get('stock') amount = self.kwargs.get('amount', 5000.0) user = self.request.user validate_stock(stock) validate_user(user) quote, latest_date = latest_quote(stock) cur_price = quote.last_price if cur_price >= 1e-6: return { 'asx_code': stock, 'user': user, 'buy_date': latest_date, 'price_at_buy_date': cur_price, 'amount': amount, 'n': int(amount / cur_price) } else: warning(self.request, "Cannot buy {} as its price is zero/unknown".format(stock)) return {}
def show_stock_sector(request, stock): validate_stock(stock) validate_user(request.user) _, company_details = stock_info(stock, lambda msg: warning(request, msg)) sector = company_details.sector_name if company_details else None all_stocks_cip = cached_all_stocks_cip(Timeframe(past_n_days=180)) # invoke separate function to cache the calls when we can c_vs_s_plot, sector_momentum_plot, sector_companies = analyse_sector_performance( stock, sector, all_stocks_cip) point_score_plot = net_rule_contributors_plot = None if sector_companies is not None: point_score_plot, net_rule_contributors_plot = \ plot_point_scores(stock, sector_companies, all_stocks_cip, default_point_score_rules()) context = { "is_sector": True, "asx_code": stock, "sector_momentum_plot": sector_momentum_plot, "sector_momentum_title": "{} sector stocks".format(sector), "company_versus_sector_plot": c_vs_s_plot, "company_versus_sector_title": "{} vs. {} performance".format(stock, sector), "point_score_plot": point_score_plot, "point_score_plot_title": "Points score due to price movements", "net_contributors_plot": net_rule_contributors_plot, "net_contributors_plot_title": "Contributions to point score by rule", } return render(request, "stock_sector.html", context)
def show_stock(request, stock=None, n_days=2 * 365): """ Displays a view of a single stock via the stock_view.html template and associated state """ validate_stock(stock) validate_user(request.user) timeframe = Timeframe( past_n_days=n_days + 200 ) # add 200 days so MA 200 can initialise itself before the plotting starts... stock_df = rsi_data( stock, timeframe) # may raise 404 if too little data available securities, company_details = stock_info(stock, lambda msg: warning(request, msg)) momentum_plot = make_rsi_plot(stock, stock_df) # plot the price over timeframe in monthly blocks prices = stock_df[[ 'last_price' ]].transpose() # use list of columns to ensure pd.DataFrame not pd.Series #print(prices) monthly_maximum_plot = plot_trend(prices, sample_period='M') # populate template and render HTML page with context context = { "asx_code": stock, "securities": securities, "cd": company_details, "rsi_plot": momentum_plot, "is_momentum": True, "monthly_highest_price_plot_title": "Maximum price each month trend", "monthly_highest_price_plot": monthly_maximum_plot, "timeframe": f"{n_days} days", "watched": user_watchlist(request.user), } return render(request, "stock_view.html", context=context)
def show_stock(request, stock=None, sector_n_days=90): """ Displays a view of a single stock via the stock_view.html template and associated state """ validate_stock(stock) validate_user(request.user) window_size = 14 # since must have a full window before computing momentum over sector_n_days all_dates = desired_dates(start_date=sector_n_days + window_size) wanted_fields = [ 'last_price', 'volume', 'day_low_price', 'day_high_price', 'eps', 'pe', 'annual_dividend_yield' ] stock_df = company_prices([stock], all_dates=all_dates, fields=wanted_fields) #print(stock_df) securities = Security.objects.filter(asx_code=stock) company_details = CompanyDetails.objects.filter(asx_code=stock).first() if company_details is None: warning(request, "No details available for {}".format(stock)) n_dates = len(stock_df) if n_dates < 14: # RSI requires at least 14 prices to plot so reject recently added stocks raise Http404("Insufficient price quotes for {} - only {}".format( stock, n_dates)) # plot relative strength fig = make_rsi_plot(stock, stock_df) # show sector performance over past 3 months all_stocks_cip = company_prices(None, all_dates=all_dates, fields='change_in_percent', fix_missing=False) sector = company_details.sector_name if company_details else None t = analyse_sector(stock, sector, all_stocks_cip, window_size=window_size) c_vs_s_plot, sector_momentum_plot, point_score_plot = t # key indicator performance over past 90 days (for now): pe, eps, yield etc. key_indicator_plot = plot_key_stock_indicators(stock_df, stock) # plot the price over last 600 days in monthly blocks ie. max 24 bars which is still readable monthly_maximum_plot = plot_best_monthly_price_trend( all_quotes(stock, all_dates=desired_dates(start_date=600))) # populate template and render HTML page with context context = { 'rsi_data': fig, 'asx_code': stock, 'securities': securities, 'cd': company_details, 'sector_momentum_plot': sector_momentum_plot, 'sector_momentum_title': "{} sector stocks: {} day performance".format(sector, sector_n_days), 'company_versus_sector_plot': c_vs_s_plot, 'company_versus_sector_title': '{} vs. {} performance'.format(stock, sector), 'key_indicators_plot': key_indicator_plot, 'monthly_highest_price_plot_title': 'Maximum price each month trend', 'monthly_highest_price_plot': monthly_maximum_plot, 'point_score_plot': point_score_plot, 'point_score_plot_title': 'Points score due to price movements' } return render(request, "stock_view.html", context=context)
def company_prices(stock_codes, all_dates=None, fields='last_price', fail_missing_months=True, fix_missing=True): """ Return a dataframe with the required companies (iff quoted) over the specified dates. By default last_price is provided. Fields may be a list, in which case the dataframe has columns for each field and dates are rows (in this case only one stock is permitted) """ if not isinstance(fields, str): # assume iterable if not str... assert len(stock_codes) == 1 dataframes = [ company_prices(stock_codes, all_dates=all_dates, fields=field, fail_missing_months=fail_missing_months) for field in fields ] result_df = pd.concat(dataframes, ignore_index=True) result_df.set_index(pd.Index(fields), inplace=True) #print(result_df) result_df = result_df.transpose() #print(result_df) assert list(result_df.columns) == fields # reject rows which are all NA to avoid downstream problems eg. plotting stocks # NB: we ONLY do this for the multi-field case, single field it is callers responsibility result_df = result_df.dropna() return result_df #print(stock_codes) assert isinstance(fields, str) if all_dates is None: all_dates = [datetime.strftime(datetime.now(), "%Y-%m-%d")] required_tags = set() for date in all_dates: validate_date(date) yyyy = date[0:4] mm = date[5:7] required_tags.add("{}-{}-{}-asx".format(fields, mm, yyyy)) which_cols = set(all_dates) # construct a "super" dataframe from the constituent parquet data superdf, n_dataframes = make_superdf(required_tags, stock_codes) # drop columns not present in all_dates to ensure we are giving just the results requested cols_to_drop = [date for date in superdf.columns if date not in which_cols] superdf = superdf.drop(columns=cols_to_drop) # on the first of the month, we dont have data yet so we permit one missing tag for this reason if fail_missing_months and n_dataframes < len(required_tags) - 1: raise ValueError( "Not all required data is available - aborting! Found {} wanted {}" .format(n_dataframes, required_tags)) # NB: ensure all columns are ALWAYS in ascending date order dates = sorted(list(superdf.columns), key=lambda k: datetime.strptime(k, "%Y-%m-%d")) superdf = superdf[dates] if fix_missing and superdf.isnull().values.any(): warning( None, "Missing data found in fields={} stocks={} over dates: {}-{}". format(fields, stock_codes, all_dates[0], all_dates[-1])) superdf = impute_missing(superdf) return superdf
def show_companies( matching_companies, # may be QuerySet or iterable of stock codes (str) request, sentiment_timeframe: Timeframe, extra_context=None, template_name="all_stocks.html", ): """ Support function to public-facing views to eliminate code redundancy """ virtual_purchases_by_user = user_purchases(request.user) if isinstance(matching_companies, QuerySet): stocks_queryset = matching_companies # we assume QuerySet is already sorted by desired criteria elif matching_companies is None or len(matching_companies) > 0: stocks_queryset, _ = latest_quote(matching_companies) # FALLTHRU # sort queryset as this will often be requested by the USER sort_by = tuple(request.GET.get("sort_by", "asx_code").split(",")) info(request, "Sorting by {}".format(sort_by)) stocks_queryset = stocks_queryset.order_by(*sort_by) # keep track of stock codes for template convenience asx_codes = [quote.asx_code for quote in stocks_queryset.all()] n_top_bottom = extra_context['n_top_bottom'] if 'n_top_bottom' in extra_context else 20 print("show_companies: found {} stocks".format(len(asx_codes))) # setup context dict for the render context = { # NB: title and heatmap_title are expected to be supplied by caller via extra_context "timeframe": sentiment_timeframe, "title": "Caller must override", "watched": user_watchlist(request.user), "n_stocks": len(asx_codes), "n_top_bottom": n_top_bottom, "virtual_purchases": virtual_purchases_by_user, } # since we sort above, we must setup the pagination also... assert isinstance(stocks_queryset, QuerySet) paginator = Paginator(stocks_queryset, 50) page_number = request.GET.get("page", 1) page_obj = paginator.page(page_number) context['page_obj'] = page_obj context['object_list'] = paginator if len(asx_codes) <= 0: warning(request, "No matching companies found.") else: df = selected_cached_stocks_cip(asx_codes, sentiment_timeframe) sentiment_heatmap_data, top10, bottom10 = plot_heatmap(df, sentiment_timeframe, n_top_bottom=n_top_bottom) sector_breakdown_plot = plot_breakdown(df) context.update({ "best_ten": top10, "worst_ten": bottom10, "sentiment_heatmap": sentiment_heatmap_data, "sentiment_heatmap_title": "{}: {}".format(context['title'], sentiment_timeframe.description), "sector_breakdown_plot": sector_breakdown_plot, }) if extra_context: context.update(extra_context) add_messages(request, context) #print(context) return render(request, template_name, context=context)
def show_stock(request, stock=None, n_days=2 * 365): """ Displays a view of a single stock via the template and associated state """ validate_stock(stock) validate_user(request.user) plot_timeframe = Timeframe(past_n_days=n_days) # for template def dataframe(ld: LazyDictionary) -> pd.DataFrame: momentum_timeframe = Timeframe( past_n_days=n_days + 200 ) # to warmup MA200 function df = company_prices( (stock,), momentum_timeframe, fields=all_stock_fundamental_fields, missing_cb=None, ) return df # key dynamic images and text for HTML response. We only compute the required data if image(s) not cached # print(df) ld = LazyDictionary() ld["stock_df"] = lambda ld: ld["stock_df_200"].filter( items=plot_timeframe.all_dates(), axis="rows" ) ld["cip_df"] = lambda: cached_all_stocks_cip(plot_timeframe) ld["stock_df_200"] = lambda ld: dataframe(ld) ld["sector_companies"] = lambda: companies_with_same_sector(stock) ld["company_details"] = lambda: stock_info(stock, lambda msg: warning(request, msg)) ld["sector"] = lambda ld: ld["company_details"].get("sector_name", "") # point_score_results is a tuple (point_score_df, net_points_by_rule) ld["point_score_results"] = lambda ld: make_point_score_dataframe( stock, default_point_score_rules(), ld ) ld["stock_vs_sector_df"] = lambda ld: make_stock_vs_sector_dataframe( ld["cip_df"], stock, ld["sector_companies"] ) print(ld["stock_vs_sector_df"]) momentum_plot = cache_plot( f"{plot_timeframe.description}-{stock}-rsi-plot", lambda ld: plot_momentum(stock, plot_timeframe, ld), datasets=ld, ) monthly_maximum_plot = cache_plot( f"{plot_timeframe.description}-{stock}-monthly-maximum-plot", lambda ld: plot_trend("M", ld), datasets=ld, ) monthly_returns_plot = cache_plot( f"{plot_timeframe.description}-{stock}-monthly returns", lambda ld: plot_monthly_returns(plot_timeframe, stock, ld), datasets=ld, ) company_versus_sector_plot = cache_plot( f"{stock}-{ld['sector']}-company-versus-sector", lambda ld: plot_company_versus_sector( ld["stock_vs_sector_df"], stock, ld["sector"] ), datasets=ld, ) point_score_plot = cache_plot( f"{plot_timeframe.description}-{stock}-point-score-plot", lambda ld: plot_series(ld["point_score_results"][0], x="date", y="points"), datasets=ld, ) net_rule_contributors_plot = cache_plot( f"{plot_timeframe.description}-{stock}-rules-by-points", lambda ld: plot_points_by_rule(ld["point_score_results"][1]), datasets=ld, ) # populate template and render HTML page with context context = { "asx_code": stock, "watched": user_watchlist(request.user), "timeframe": plot_timeframe, "information": ld["company_details"], "momentum": { "rsi_plot": momentum_plot, "monthly_highest_price": { "title": "Highest price each month", "plot_uri": monthly_maximum_plot, }, }, "fundamentals": { "plot_uri": cache_plot( f"{stock}-{plot_timeframe.description}-fundamentals-plot", lambda ld: plot_fundamentals( fundamentals_dataframe(plot_timeframe, stock, ld), stock, ), datasets=ld, ), "title": "Stock fundamentals: EPS, PE, DY etc.", "timeframe": plot_timeframe, }, "stock_vs_sector": { "plot_uri": company_versus_sector_plot, "title": "Company versus sector - percentage change", "timeframe": plot_timeframe, }, "point_score": { "plot_uri": point_score_plot, "title": "Points score due to price movements", }, "net_contributors": { "plot_uri": net_rule_contributors_plot, "title": "Contributions to point score by rule", }, "month_by_month_return_uri": monthly_returns_plot, } return render(request, "stock_page.html", context=context)
def show_companies( matching_companies, # may be QuerySet or iterable of stock codes (str) request, sentiment_timeframe: Timeframe, extra_context=None, template_name="all_stocks.html", ): """ Support function to public-facing views to eliminate code redundancy """ if isinstance(matching_companies, QuerySet): stocks_queryset = matching_companies # we assume QuerySet is already sorted by desired criteria elif matching_companies is None or len(matching_companies) > 0: stocks_queryset, _ = latest_quote(matching_companies) # FALLTHRU else: # no companies to report? warning(request, "No matching companies.") return render(request, template_name, context={"timeframe": sentiment_timeframe}) # prune companies without a latest price, makes no sense to report them stocks_queryset = stocks_queryset.exclude(last_price__isnull=True) # sort queryset as this will often be requested by the USER arg = request.GET.get("sort_by", "asx_code") #info(request, "Sorting by {}".format(arg)) if arg == "sector" or arg == "sector,-eps": ss = { s["asx_code"]: s["sector_name"] for s in stocks_by_sector().to_dict("records") } if arg == "sector": stocks_queryset = sorted(stocks_queryset, key=lambda s: ss.get(s.asx_code, "Z") ) # companies without sector sort last else: eps_dict = { s.asx_code: s.eps if s.eps is not None else 0.0 for s in stocks_queryset } stocks_queryset = sorted( stocks_queryset, key=lambda s: (ss.get(s.asx_code, "Z"), -eps_dict.get(s.asx_code, 0.0)), ) else: sort_by = tuple(arg.split(",")) stocks_queryset = stocks_queryset.order_by(*sort_by) # keep track of stock codes for template convenience asx_codes = [quote.asx_code for quote in stocks_queryset] n_top_bottom = (extra_context["n_top_bottom"] if "n_top_bottom" in extra_context else 20) print("show_companies: found {} stocks".format(len(asx_codes))) # setup context dict for the render context = { # NB: title and heatmap_title are expected to be supplied by caller via extra_context "timeframe": sentiment_timeframe, "title": "Caller must override", "watched": user_watchlist(request.user), "n_stocks": len(asx_codes), "n_top_bottom": n_top_bottom, "virtual_purchases": user_purchases(request.user), } # since we sort above, we must setup the pagination also... # assert isinstance(stocks_queryset, QuerySet) paginator = Paginator(stocks_queryset, 50) page_number = request.GET.get("page", 1) page_obj = paginator.page(page_number) context["page_obj"] = page_obj context["object_list"] = paginator # compute totals across all dates for the specified companies to look at top10/bottom10 in the timeframe ld = LazyDictionary() ld["cip_df"] = lambda ld: selected_cached_stocks_cip( asx_codes, sentiment_timeframe) ld["sum_by_company"] = lambda ld: ld["cip_df"].sum(axis=1, numeric_only=True) ld["top10"] = lambda ld: ld["sum_by_company"].nlargest(n_top_bottom) ld["bottom10"] = lambda ld: ld["sum_by_company"].nsmallest(n_top_bottom) ld["stocks_by_sector"] = lambda ld: stocks_by_sector() if len(asx_codes) <= 0 or len(ld["top10"]) <= 0: warning(request, "No matching companies found.") else: sorted_codes = "-".join(sorted(asx_codes)) sentiment_heatmap_uri = cache_plot( f"{sorted_codes}-{sentiment_timeframe.description}-stocks-sentiment-plot", lambda ld: plot_heatmap(sentiment_timeframe, ld), datasets=ld, ) key = f"{sorted_codes}-{sentiment_timeframe.description}-breakdown-plot" sector_breakdown_uri = cache_plot(key, plot_breakdown, datasets=ld) top10_plot_uri = cache_plot( f"top10-plot-{'-'.join(ld['top10'].index)}", lambda ld: plot_cumulative_returns(ld["top10"].index, ld), datasets=ld, ) bottom10_plot_uri = cache_plot( f"bottom10-plot-{'-'.join(ld['bottom10'].index)}", lambda ld: plot_cumulative_returns(ld["bottom10"].index, ld), datasets=ld, ) context.update({ "best_ten": ld["top10"], "worst_ten": ld["bottom10"], "sentiment_heatmap_uri": sentiment_heatmap_uri, "sentiment_heatmap_title": "{}: {}".format(context["title"], sentiment_timeframe.description), "sector_breakdown_uri": sector_breakdown_uri, "top10_plot_uri": top10_plot_uri, "bottom10_plot_uri": bottom10_plot_uri, "timeframe_end_performance": timeframe_end_performance(ld), }) if extra_context: context.update(extra_context) add_messages(request, context) # print(context) return render(request, template_name, context=context)
def show_financial_metrics(request, stock=None): validate_user(request.user) validate_stock(stock) def data_factory(ld: LazyDictionary): data_df = financial_metrics(stock) if data_df is None or len(data_df) < 1: raise Http404(f"No financial metrics available for {stock}") return data_df def find_linear_metrics(ld: LazyDictionary) -> Iterable[str]: linear_metrics = calculate_trends(ld["data_df"]) good_linear_metrics = [] for k, t in linear_metrics.items(): if t[1] < 0.1: good_linear_metrics.append(k) return good_linear_metrics def find_exp_metrics(ld: LazyDictionary) -> Iterable[str]: exp_metrics = calculate_trends( ld["data_df"], polynomial_degree=2, nrmse_cutoff=0.05 ) good_linear_metrics = set(ld["linear_metrics"]) good_exp_metrics = [] for k, t in exp_metrics.items(): if t[1] < 0.1 and k not in good_linear_metrics: good_exp_metrics.append(k) return good_exp_metrics ld = LazyDictionary() ld["data_df"] = lambda ld: data_factory(ld) ld["linear_metrics"] = lambda ld: find_linear_metrics(ld) ld["exp_metrics"] = lambda ld: find_exp_metrics(ld) # print( # f"n_metrics == {len(data_df)} n_trending={len(linear_metrics.keys())} n_good_fit={len(good_linear_metrics)} n_good_exp={len(good_exp_metrics)}" # ) def plot_metrics(df: pd.DataFrame, use_short_labels=False, **kwargs): plot = ( p9.ggplot(df, p9.aes(x="date", y="value", colour="metric")) + p9.geom_line(size=1.3) + p9.geom_point(size=3) ) if use_short_labels: plot += p9.scale_y_continuous(labels=label_shorten) n_metrics = df["metric"].nunique() return user_theme( plot, subplots_adjust={"left": 0.2}, figure_size=(12, int(n_metrics * 1.5)), **kwargs, ) def plot_linear_trending_metrics(ld: LazyDictionary): df = ld["data_df"].filter(ld["linear_metrics"], axis=0) if len(df) < 1: return None df["metric"] = df.index df = df.melt(id_vars="metric").dropna(how="any", axis=0) plot = plot_metrics(df, use_short_labels=True) plot += p9.facet_wrap("~metric", ncol=1, scales="free_y") return plot def plot_exponential_growth_metrics(ld: LazyDictionary): df = ld["data_df"].filter(ld["exp_metrics"], axis=0) if len(df) < 1: return None df["metric"] = df.index df = df.melt(id_vars="metric").dropna(how="any", axis=0) plot = plot_metrics(df) plot += p9.facet_wrap("~metric", ncol=1, scales="free_y") return plot def plot_earnings_and_revenue(ld: LazyDictionary): df = ld["data_df"].filter(["Ebit", "Total Revenue", "Earnings"], axis=0) if len(df) < 2: print(f"WARNING: revenue and earnings not availabe for {stock}") return None df["metric"] = df.index df = df.melt(id_vars="metric").dropna(how="any", axis=0) plot = plot_metrics( df, use_short_labels=True, legend_position="right", y_axis_label="$ AUD", ) # need to show metric name somewhere on plot return plot er_uri = cache_plot( f"{stock}-earnings-revenue-plot", lambda ld: plot_earnings_and_revenue(ld), datasets=ld, ) trending_metrics_uri = cache_plot( f"{stock}-trending-metrics-plot", lambda ld: plot_linear_trending_metrics(ld), datasets=ld, ) exp_growth_metrics_uri = cache_plot( f"{stock}-exponential-growth-metrics-plot", lambda ld: plot_exponential_growth_metrics(ld), datasets=ld, ) warning( request, "Due to experimental data ingest - data on this page may be wrong/misleading/inaccurate/missing. Use at own risk.", ) context = { "asx_code": stock, "data": ld["data_df"], "earnings_and_revenue_plot_uri": er_uri, "trending_metrics_plot_uri": trending_metrics_uri, "exp_growth_metrics_plot_uri": exp_growth_metrics_uri, } return render(request, "stock_financial_metrics.html", context=context)
def detect_outliers(stocks: list, all_stocks_cip: pd.DataFrame, rules=None): """ Returns a dataframe describing those outliers present in stocks based on the provided rules. All_stocks_cip is the "change in percent" for at least the stocks present in the specified list """ if rules is None: rules = default_point_score_rules() str_rules = {str(r): r for r in rules} rows = [] stocks_by_sector_df = (stocks_by_sector() ) # NB: ETFs in watchlist will have no sector stocks_by_sector_df.index = stocks_by_sector_df["asx_code"] for stock in stocks: # print("Processing stock: ", stock) try: sector = stocks_by_sector_df.at[stock, "sector_name"] sector_companies = list(stocks_by_sector_df.loc[ stocks_by_sector_df["sector_name"] == sector].asx_code) # day_low_high() may raise KeyError when data is currently being fetched, so it appears here... day_low_high_df = day_low_high(stock, all_stocks_cip.columns) except KeyError: warning( None, "Unable to locate watchlist entry: {} - continuing without it". format(stock), ) continue state = { "day_low_high_df": day_low_high_df, # never changes each day, so we init it here "all_stocks_change_in_percent_df": all_stocks_cip, "stock": stock, "daily_range_threshold": 0.20, # 20% at either end of the daily range gets a point } points_by_rule = defaultdict(int) for date in all_stocks_cip.columns: market_avg = all_stocks_cip[date].mean() sector_avg = all_stocks_cip[date].filter( items=sector_companies).mean() stock_move = all_stocks_cip.at[stock, date] state.update({ "market_avg": market_avg, "sector_avg": sector_avg, "stock_move": stock_move, "date": date, }) for rule_name, rule in str_rules.items(): try: points_by_rule[rule_name] += rule(state) except TypeError: # handle nan's in dataset safely pass d = {"stock": stock} d.update(points_by_rule) rows.append(d) df = pd.DataFrame.from_records(rows) df = df.set_index("stock") # print(df) clf = IForest() clf.fit(df) scores = clf.predict(df) results = [ row[0] for row, value in zip(df.iterrows(), scores) if value > 0 ] # print(results) print("Found {} outlier stocks".format(len(results))) return results