def test_user_watchlist( uw_fixture, django_user_model ): # pylint: disable=unused-argument,redefined-outer-name u1 = django_user_model.objects.get(username="******") assert user_watchlist(u1) == set(["ASX1"]) u2 = django_user_model.objects.get(username="******") assert user_watchlist(u2) == set()
def show_trends(request): user = request.user validate_user(user) stocks = user_watchlist(user) timeframe = Timeframe(past_n_days=300) ld = LazyDictionary() ld["cip_df"] = lambda ld: selected_cached_stocks_cip(stocks, timeframe) ld["trends"] = lambda ld: calculate_trends(ld["cip_df"]) ld["rank"] = lambda ld: rank_cumulative_change( ld["cip_df"].filter(ld["trends"].keys(), axis="index"), timeframe ) trending_companies_plot = cache_plot( f"{user.username}-watchlist-trends", lambda ld: plot_company_rank(ld), datasets=ld, ) context = { "watchlist_trends": ld["trends"], "timeframe": timeframe, "trending_companies_uri": trending_companies_plot, "trending_companies_plot_title": "Trending watchlist stocks (ranked): {}".format( timeframe.description ), } return render(request, "watchlist-rank.html", context=context)
def test_toggle_watchlist_entry(uw_fixture, django_user_model): u = find_user('u2') assert u is not None uname = u.username assert not is_in_watchlist(uname, 'ASX1') toggle_watchlist_entry(u, 'ASX1') assert is_in_watchlist(uname, 'ASX1') assert not is_in_watchlist(uname, 'ASX2') toggle_watchlist_entry(u, 'ASX2') ret = user_watchlist(u) assert 'ASX2' in ret
def get_dataset(dataset_wanted, request, timeframe=None): assert (dataset_wanted in set(["market_sentiment", "eps-per-sector"]) or dataset_wanted.startswith("kmeans-") or dataset_wanted.startswith("financial-metrics-") or dataset_wanted.startswith("stock-quotes-")) if timeframe is None: timeframe = Timeframe(past_n_days=300) if dataset_wanted == "market_sentiment": df = cached_all_stocks_cip(timeframe) return df elif dataset_wanted == "kmeans-watchlist": _, _, _, _, df = make_kmeans_cluster_dataframe( timeframe, 7, user_watchlist(request.user)) return df elif dataset_wanted == "kmeans-etfs": _, _, _, _, df = make_kmeans_cluster_dataframe(timeframe, 7, all_etfs()) return df elif dataset_wanted.startswith("stock-quotes-"): stock = dataset_wanted[len("stock-quotes-"):] validate_stock(stock) df = company_prices([stock], timeframe=timeframe, fields=all_stock_fundamental_fields, missing_cb=None) df['stock_code'] = stock return df elif dataset_wanted.startswith("kmeans-sector-"): sector_id = int(dataset_wanted[14:]) sector = Sector.objects.get(sector_id=sector_id) if sector is None or sector.sector_name is None: raise Http404("No stocks associated with sector") asx_codes = all_sector_stocks(sector.sector_name) _, _, _, _, df = make_kmeans_cluster_dataframe(timeframe, 7, asx_codes) return df elif dataset_wanted.startswith("financial-metrics-"): stock = dataset_wanted[len("financial-metrics-"):] validate_stock(stock) df = financial_metrics(stock) if df is not None: # excel doesnt support timezones, so we remove it first colnames = [d.strftime("%Y-%m-%d") for d in df.columns] df.columns = colnames # FALLTHRU return df elif dataset_wanted == "eps-per-sector": df, _ = pe_trends_df(Timeframe(past_n_days=180)) df = make_pe_trends_eps_df(df, stocks_by_sector()) df = df.set_index("asx_code", drop=True) return df else: raise ValueError("Unsupported dataset {}".format(dataset_wanted))
def filter_stocks_to_search(request, what_to_search: str) -> set: assert request is not None assert len(what_to_search) > 0 if what_to_search == "all_stocks": stocks_to_consider = all_stocks() elif what_to_search == "watchlist": stocks_to_consider = user_watchlist(request.user) else: #print(what_to_search) stocks_to_consider = all_sector_stocks(what_to_search) return stocks_to_consider
def show_watched(request): validate_user(request.user) matching_companies = user_watchlist(request.user) timeframe = Timeframe() return show_companies( matching_companies, request, timeframe, { "title": "Stocks you are watching", "sentiment_heatmap_title": "Watchlist stocks sentiment: {}".format(timeframe.description), })
def show_watched(request): validate_user(request.user) matching_companies = user_watchlist(request.user) timeframe = Timeframe() return show_companies( matching_companies, request, timeframe, { "title": "Stocks you are watching", "sentiment_heatmap_title": "Watchlist sentiment heatmap", }, )
def test_toggle_watchlist_entry( uw_fixture, django_user_model ): # pylint: disable=unused-argument,redefined-outer-name global watchlist_cache u = find_user("u2") assert u is not None uname = u.username mdl.watchlist_cache.clear() assert not is_in_watchlist(uname, "ASX1") toggle_watchlist_entry(u, "ASX1") mdl.watchlist_cache.clear() assert is_in_watchlist(uname, "ASX1") assert not is_in_watchlist(uname, "ASX2") toggle_watchlist_entry(u, "ASX2") user_watchlist.cache_clear() ret = user_watchlist(u) assert "ASX2" in ret
def market_sentiment(request, n_days=21, n_top_bottom=20, sector_n_days=180): validate_user(request.user) assert n_days > 0 assert n_top_bottom > 0 timeframe = Timeframe(past_n_days=n_days) sector_timeframe = Timeframe(past_n_days=sector_n_days) df = cached_all_stocks_cip(timeframe) sector_df = cached_all_stocks_cip(sector_timeframe) sentiment_plot, top10, bottom10 = plot_heatmap(df, timeframe, n_top_bottom=n_top_bottom) sector_performance_plot = plot_market_wide_sector_performance(sector_df) context = { "sentiment_data": sentiment_plot, "n_days": timeframe.n_days, "n_stocks_plotted": len(df), "n_top_bottom": n_top_bottom, "best_ten": top10, "worst_ten": bottom10, "watched": user_watchlist(request.user), "sector_performance": sector_performance_plot, "sector_performance_title": "Cumulative sector avg. performance: {}".format( sector_timeframe.description), "title": "Market sentiment: {}".format(timeframe.description), "market_cap_distribution_plot": plot_market_cap_distribution(tuple(df.index), latest_quotation_date('ANZ'), sector_df.columns[0]) } return render(request, "market_sentiment_view.html", context=context)
def show_trends(request): validate_user(request.user) watchlist_stocks = user_watchlist(request.user) timeframe = Timeframe(past_n_days=300) cip = selected_cached_stocks_cip(watchlist_stocks, timeframe) trends = calculate_trends(cip, watchlist_stocks) #print(trends) # for now we only plot trending companies... too slow and unreadable to load the page otherwise! cip = rank_cumulative_change(cip.filter(trends.keys(), axis="index"), timeframe) #print(cip) trending_companies_plot = plot_company_rank(cip) context = { "watchlist_trends": trends, "trending_companies_plot": trending_companies_plot, "trending_companies_plot_title": "Trending watchlist companies by rank: {}".format( timeframe.description), } return render(request, "trends.html", context=context)
def show_stock(request, stock=None, n_days=2 * 365): """ Displays a view of a single stock via the stock_view.html template and associated state """ validate_stock(stock) validate_user(request.user) timeframe = Timeframe( past_n_days=n_days + 200 ) # add 200 days so MA 200 can initialise itself before the plotting starts... stock_df = rsi_data( stock, timeframe) # may raise 404 if too little data available securities, company_details = stock_info(stock, lambda msg: warning(request, msg)) momentum_plot = make_rsi_plot(stock, stock_df) # plot the price over timeframe in monthly blocks prices = stock_df[[ 'last_price' ]].transpose() # use list of columns to ensure pd.DataFrame not pd.Series #print(prices) monthly_maximum_plot = plot_trend(prices, sample_period='M') # populate template and render HTML page with context context = { "asx_code": stock, "securities": securities, "cd": company_details, "rsi_plot": momentum_plot, "is_momentum": True, "monthly_highest_price_plot_title": "Maximum price each month trend", "monthly_highest_price_plot": monthly_maximum_plot, "timeframe": f"{n_days} days", "watched": user_watchlist(request.user), } return render(request, "stock_view.html", context=context)
def market_sentiment(request, n_days=21, n_top_bottom=20, sector_n_days=365): validate_user(request.user) assert n_days > 0 assert n_top_bottom > 0 def market_cap_data_factory(ld: LazyDictionary) -> pd.DataFrame: dates = ld["sector_timeframe"].all_dates() # print(dates) assert len(dates) > 90 result_df = None adjusted_dates = [] for the_date in [dates[0], dates[-1], dates[-30], dates[-90]]: print(f"Before valid_quotes_only for {the_date}") quotes, actual_trading_date = valid_quotes_only( the_date, ensure_date_has_data=True) print(f"After valid_quotes_only for {the_date}") print(f"Before make quotes {actual_trading_date}") print(len(quotes)) df = make_quote_df(quotes, ld["asx_codes"], actual_trading_date) print("After make_quote_df") result_df = df if result_df is None else result_df.append(df) if the_date != actual_trading_date: adjusted_dates.append(the_date) if len(adjusted_dates) > 0: warning( request, "Some dates were not trading days, adjusted: {}".format( adjusted_dates), ) return result_df ld = LazyDictionary() ld["asx_codes"] = lambda ld: all_stocks() ld["sector_timeframe"] = lambda ld: Timeframe(past_n_days=sector_n_days) ld["timeframe"] = lambda ld: Timeframe(past_n_days=n_days) ld["sector_df"] = lambda ld: cached_all_stocks_cip(ld["sector_timeframe"]) ld["sector_cumsum_df"] = lambda ld: ld["sector_df"].cumsum(axis=1) ld["cip_df"] = lambda ld: ld["sector_df"].filter( items=ld["timeframe"].all_dates(), axis=1) ld["market_cap_df"] = lambda ld: market_cap_data_factory(ld) ld["stocks_by_sector"] = lambda ld: stocks_by_sector() sentiment_plot = cache_plot( f"market-sentiment-{ld['timeframe'].description}", lambda ld: plot_heatmap(ld["timeframe"], ld), datasets=ld, ) sector_descr = ld["sector_timeframe"].description sector_performance_plot = cache_plot( f"sector-performance-{sector_descr}", lambda ld: plot_market_wide_sector_performance(ld), datasets=ld, ) market_cap_dist_plot = cache_plot( f"market-cap-dist-{sector_descr}", lambda ld: plot_market_cap_distribution(ld), datasets=ld, ) df = ld["sector_cumsum_df"].transpose() df.index = pd.to_datetime(df.index, format="%Y-%m-%d") df = (df.resample("BM", ).asfreq().diff(periods=1)) ld["monthly_returns_by_stock"] = df # print(df) context = { "sentiment_uri": sentiment_plot, "n_days": ld["timeframe"].n_days, "n_stocks_plotted": len(ld["asx_codes"]), "n_top_bottom": n_top_bottom, "watched": user_watchlist(request.user), "sector_performance_uri": sector_performance_plot, "sector_timeframe": ld["sector_timeframe"], "sector_performance_title": "Cumulative sector avg. performance: {}".format( ld["sector_timeframe"].description), "title": "Market sentiment", "market_cap_distribution_uri": market_cap_dist_plot, "monthly_sector_mean_returns": plot_sector_monthly_mean_returns(ld), } return render(request, "market_sentiment_view.html", context=context)
def show_companies( matching_companies, # may be QuerySet or iterable of stock codes (str) request, sentiment_timeframe: Timeframe, extra_context=None, template_name="all_stocks.html", ): """ Support function to public-facing views to eliminate code redundancy """ virtual_purchases_by_user = user_purchases(request.user) if isinstance(matching_companies, QuerySet): stocks_queryset = matching_companies # we assume QuerySet is already sorted by desired criteria elif matching_companies is None or len(matching_companies) > 0: stocks_queryset, _ = latest_quote(matching_companies) # FALLTHRU # sort queryset as this will often be requested by the USER sort_by = tuple(request.GET.get("sort_by", "asx_code").split(",")) info(request, "Sorting by {}".format(sort_by)) stocks_queryset = stocks_queryset.order_by(*sort_by) # keep track of stock codes for template convenience asx_codes = [quote.asx_code for quote in stocks_queryset.all()] n_top_bottom = extra_context['n_top_bottom'] if 'n_top_bottom' in extra_context else 20 print("show_companies: found {} stocks".format(len(asx_codes))) # setup context dict for the render context = { # NB: title and heatmap_title are expected to be supplied by caller via extra_context "timeframe": sentiment_timeframe, "title": "Caller must override", "watched": user_watchlist(request.user), "n_stocks": len(asx_codes), "n_top_bottom": n_top_bottom, "virtual_purchases": virtual_purchases_by_user, } # since we sort above, we must setup the pagination also... assert isinstance(stocks_queryset, QuerySet) paginator = Paginator(stocks_queryset, 50) page_number = request.GET.get("page", 1) page_obj = paginator.page(page_number) context['page_obj'] = page_obj context['object_list'] = paginator if len(asx_codes) <= 0: warning(request, "No matching companies found.") else: df = selected_cached_stocks_cip(asx_codes, sentiment_timeframe) sentiment_heatmap_data, top10, bottom10 = plot_heatmap(df, sentiment_timeframe, n_top_bottom=n_top_bottom) sector_breakdown_plot = plot_breakdown(df) context.update({ "best_ten": top10, "worst_ten": bottom10, "sentiment_heatmap": sentiment_heatmap_data, "sentiment_heatmap_title": "{}: {}".format(context['title'], sentiment_timeframe.description), "sector_breakdown_plot": sector_breakdown_plot, }) if extra_context: context.update(extra_context) add_messages(request, context) #print(context) return render(request, template_name, context=context)
def stocks(self): return list(user_watchlist(self.request.user))
def show_watchlist_outliers(request, n_days=30): validate_user(request.user) stocks = user_watchlist(request.user) return show_outliers(request, stocks, n_days=n_days)
def show_companies( matching_companies, # may be QuerySet or iterable of stock codes (str) request, sentiment_timeframe: Timeframe, extra_context=None, template_name="all_stocks.html", ): """ Support function to public-facing views to eliminate code redundancy """ if isinstance(matching_companies, QuerySet): stocks_queryset = matching_companies # we assume QuerySet is already sorted by desired criteria elif matching_companies is None or len(matching_companies) > 0: stocks_queryset, _ = latest_quote(matching_companies) # FALLTHRU else: # no companies to report? warning(request, "No matching companies.") return render(request, template_name, context={"timeframe": sentiment_timeframe}) # prune companies without a latest price, makes no sense to report them stocks_queryset = stocks_queryset.exclude(last_price__isnull=True) # sort queryset as this will often be requested by the USER arg = request.GET.get("sort_by", "asx_code") #info(request, "Sorting by {}".format(arg)) if arg == "sector" or arg == "sector,-eps": ss = { s["asx_code"]: s["sector_name"] for s in stocks_by_sector().to_dict("records") } if arg == "sector": stocks_queryset = sorted(stocks_queryset, key=lambda s: ss.get(s.asx_code, "Z") ) # companies without sector sort last else: eps_dict = { s.asx_code: s.eps if s.eps is not None else 0.0 for s in stocks_queryset } stocks_queryset = sorted( stocks_queryset, key=lambda s: (ss.get(s.asx_code, "Z"), -eps_dict.get(s.asx_code, 0.0)), ) else: sort_by = tuple(arg.split(",")) stocks_queryset = stocks_queryset.order_by(*sort_by) # keep track of stock codes for template convenience asx_codes = [quote.asx_code for quote in stocks_queryset] n_top_bottom = (extra_context["n_top_bottom"] if "n_top_bottom" in extra_context else 20) print("show_companies: found {} stocks".format(len(asx_codes))) # setup context dict for the render context = { # NB: title and heatmap_title are expected to be supplied by caller via extra_context "timeframe": sentiment_timeframe, "title": "Caller must override", "watched": user_watchlist(request.user), "n_stocks": len(asx_codes), "n_top_bottom": n_top_bottom, "virtual_purchases": user_purchases(request.user), } # since we sort above, we must setup the pagination also... # assert isinstance(stocks_queryset, QuerySet) paginator = Paginator(stocks_queryset, 50) page_number = request.GET.get("page", 1) page_obj = paginator.page(page_number) context["page_obj"] = page_obj context["object_list"] = paginator # compute totals across all dates for the specified companies to look at top10/bottom10 in the timeframe ld = LazyDictionary() ld["cip_df"] = lambda ld: selected_cached_stocks_cip( asx_codes, sentiment_timeframe) ld["sum_by_company"] = lambda ld: ld["cip_df"].sum(axis=1, numeric_only=True) ld["top10"] = lambda ld: ld["sum_by_company"].nlargest(n_top_bottom) ld["bottom10"] = lambda ld: ld["sum_by_company"].nsmallest(n_top_bottom) ld["stocks_by_sector"] = lambda ld: stocks_by_sector() if len(asx_codes) <= 0 or len(ld["top10"]) <= 0: warning(request, "No matching companies found.") else: sorted_codes = "-".join(sorted(asx_codes)) sentiment_heatmap_uri = cache_plot( f"{sorted_codes}-{sentiment_timeframe.description}-stocks-sentiment-plot", lambda ld: plot_heatmap(sentiment_timeframe, ld), datasets=ld, ) key = f"{sorted_codes}-{sentiment_timeframe.description}-breakdown-plot" sector_breakdown_uri = cache_plot(key, plot_breakdown, datasets=ld) top10_plot_uri = cache_plot( f"top10-plot-{'-'.join(ld['top10'].index)}", lambda ld: plot_cumulative_returns(ld["top10"].index, ld), datasets=ld, ) bottom10_plot_uri = cache_plot( f"bottom10-plot-{'-'.join(ld['bottom10'].index)}", lambda ld: plot_cumulative_returns(ld["bottom10"].index, ld), datasets=ld, ) context.update({ "best_ten": ld["top10"], "worst_ten": ld["bottom10"], "sentiment_heatmap_uri": sentiment_heatmap_uri, "sentiment_heatmap_title": "{}: {}".format(context["title"], sentiment_timeframe.description), "sector_breakdown_uri": sector_breakdown_uri, "top10_plot_uri": top10_plot_uri, "bottom10_plot_uri": bottom10_plot_uri, "timeframe_end_performance": timeframe_end_performance(ld), }) if extra_context: context.update(extra_context) add_messages(request, context) # print(context) return render(request, template_name, context=context)
def cluster_stocks_view(request, stocks: str): """ ref: https://pythonforfinance.net/2018/02/08/stock-clusters-using-k-means-algorithm-in-python/ """ validate_user(request.user) timeframe = Timeframe(past_n_days=300) if stocks == "watchlist": asx_codes = user_watchlist(request.user) elif stocks == "etfs": asx_codes = all_etfs() elif stocks.startswith("sector-"): sector_id = int(stocks[7:]) sector = Sector.objects.get(sector_id=sector_id) if sector is None or sector.sector_name is None: raise Http404("No stocks associated with sector") asx_codes = all_sector_stocks(sector.sector_name) else: raise Http404("Unknown stock list {}".format(stocks)) chosen_k = 7 # often a reasonable tradeoff def elbow_curve_plot(ld: LazyDictionary): distortion, _, _, _, _ = make_kmeans_cluster_dataframe( timeframe, chosen_k, asx_codes ) fig = plt.figure(figsize=(15, 5)) plt.plot(range(2, 20), distortion) plt.grid(True) plt.title("Elbow curve") return fig def cluster_plot(ld: LazyDictionary): _, _, centroids, idx, data_df = make_kmeans_cluster_dataframe( timeframe, chosen_k, asx_codes ) centroids_df = pd.DataFrame.from_records( centroids, columns=["return", "volatility"] ) plot = ( p9.ggplot( data_df, p9.aes("return", "volatility", colour="factor(cluster_id)") ) + p9.geom_point(size=3) + p9.facet_wrap("~cluster_id", ncol=3, scales="free") ) return user_theme( plot, x_axis_label="Returns (%)", y_axis_label="Volatility (%)", figure_size=(15, 15), subplots_adjust={"hspace": 0.15, "wspace": 0.15}, ) stocks_as_str = "-".join(sorted(asx_codes)) elbow_curve_uri = cache_plot( f"{request.user.username}-cluster-{stocks_as_str}-elbow-curve-plot", elbow_curve_plot, ) cluster_uri = cache_plot( f"{request.user.username}-cluster-{stocks_as_str}-kmeans-cluster-plot", cluster_plot, ) context = { "elbow_curve_plot_uri": elbow_curve_uri, "k": chosen_k, "dataset": stocks, "n_stocks": len(asx_codes), "cluster_plot_uri": cluster_uri, "timeframe": timeframe, } return render(request, "cluster_stocks.html", context=context)
def test_user_watchlist(uw_fixture, django_user_model): u1 = django_user_model.objects.get(username='******') assert user_watchlist(u1) == set(['ASX1']) u2 = django_user_model.objects.get(username='******') assert user_watchlist(u2) == set()
def show_stock(request, stock=None, n_days=2 * 365): """ Displays a view of a single stock via the template and associated state """ validate_stock(stock) validate_user(request.user) plot_timeframe = Timeframe(past_n_days=n_days) # for template def dataframe(ld: LazyDictionary) -> pd.DataFrame: momentum_timeframe = Timeframe( past_n_days=n_days + 200 ) # to warmup MA200 function df = company_prices( (stock,), momentum_timeframe, fields=all_stock_fundamental_fields, missing_cb=None, ) return df # key dynamic images and text for HTML response. We only compute the required data if image(s) not cached # print(df) ld = LazyDictionary() ld["stock_df"] = lambda ld: ld["stock_df_200"].filter( items=plot_timeframe.all_dates(), axis="rows" ) ld["cip_df"] = lambda: cached_all_stocks_cip(plot_timeframe) ld["stock_df_200"] = lambda ld: dataframe(ld) ld["sector_companies"] = lambda: companies_with_same_sector(stock) ld["company_details"] = lambda: stock_info(stock, lambda msg: warning(request, msg)) ld["sector"] = lambda ld: ld["company_details"].get("sector_name", "") # point_score_results is a tuple (point_score_df, net_points_by_rule) ld["point_score_results"] = lambda ld: make_point_score_dataframe( stock, default_point_score_rules(), ld ) ld["stock_vs_sector_df"] = lambda ld: make_stock_vs_sector_dataframe( ld["cip_df"], stock, ld["sector_companies"] ) print(ld["stock_vs_sector_df"]) momentum_plot = cache_plot( f"{plot_timeframe.description}-{stock}-rsi-plot", lambda ld: plot_momentum(stock, plot_timeframe, ld), datasets=ld, ) monthly_maximum_plot = cache_plot( f"{plot_timeframe.description}-{stock}-monthly-maximum-plot", lambda ld: plot_trend("M", ld), datasets=ld, ) monthly_returns_plot = cache_plot( f"{plot_timeframe.description}-{stock}-monthly returns", lambda ld: plot_monthly_returns(plot_timeframe, stock, ld), datasets=ld, ) company_versus_sector_plot = cache_plot( f"{stock}-{ld['sector']}-company-versus-sector", lambda ld: plot_company_versus_sector( ld["stock_vs_sector_df"], stock, ld["sector"] ), datasets=ld, ) point_score_plot = cache_plot( f"{plot_timeframe.description}-{stock}-point-score-plot", lambda ld: plot_series(ld["point_score_results"][0], x="date", y="points"), datasets=ld, ) net_rule_contributors_plot = cache_plot( f"{plot_timeframe.description}-{stock}-rules-by-points", lambda ld: plot_points_by_rule(ld["point_score_results"][1]), datasets=ld, ) # populate template and render HTML page with context context = { "asx_code": stock, "watched": user_watchlist(request.user), "timeframe": plot_timeframe, "information": ld["company_details"], "momentum": { "rsi_plot": momentum_plot, "monthly_highest_price": { "title": "Highest price each month", "plot_uri": monthly_maximum_plot, }, }, "fundamentals": { "plot_uri": cache_plot( f"{stock}-{plot_timeframe.description}-fundamentals-plot", lambda ld: plot_fundamentals( fundamentals_dataframe(plot_timeframe, stock, ld), stock, ), datasets=ld, ), "title": "Stock fundamentals: EPS, PE, DY etc.", "timeframe": plot_timeframe, }, "stock_vs_sector": { "plot_uri": company_versus_sector_plot, "title": "Company versus sector - percentage change", "timeframe": plot_timeframe, }, "point_score": { "plot_uri": point_score_plot, "title": "Points score due to price movements", }, "net_contributors": { "plot_uri": net_rule_contributors_plot, "title": "Contributions to point score by rule", }, "month_by_month_return_uri": monthly_returns_plot, } return render(request, "stock_page.html", context=context)