def show_trends(request):
user = request.user
validate_user(user)
stocks = user_watchlist(user)
timeframe = Timeframe(past_n_days=300)
ld = LazyDictionary()
ld["cip_df"] = lambda ld: selected_cached_stocks_cip(stocks, timeframe)
ld["trends"] = lambda ld: calculate_trends(ld["cip_df"])
ld["rank"] = lambda ld: rank_cumulative_change(
ld["cip_df"].filter(ld["trends"].keys(), axis="index"), timeframe
)
trending_companies_plot = cache_plot(
f"{user.username}-watchlist-trends",
lambda ld: plot_company_rank(ld),
datasets=ld,
)
context = {
"watchlist_trends": ld["trends"],
"timeframe": timeframe,
"trending_companies_uri": trending_companies_plot,
"trending_companies_plot_title": "Trending watchlist stocks (ranked): {}".format(
timeframe.description
),
}
return render(request, "watchlist-rank.html", context=context)
def cached_portfolio_performance(user):
assert isinstance(user, get_user_model())
username = user.username
overall_key = f"{username}-portfolio-performance"
stock_key = f"{username}-stock-performance"
contributors_key = f"{username}-contributor-performance"
def data_factory(
ld: LazyDictionary,
): # dont create the dataframe unless we have to - avoid exxpensive call!
purchase_buy_dates = []
purchases = []
stocks = []
for stock, purchases_for_stock in user_purchases(user).items():
stocks.append(stock)
for purchase in purchases_for_stock:
purchase_buy_dates.append(purchase.buy_date)
purchases.append(purchase)
purchase_buy_dates = sorted(purchase_buy_dates)
# print("earliest {} latest {}".format(purchase_buy_dates[0], purchase_buy_dates[-1]))
timeframe = Timeframe(from_date=str(purchase_buy_dates[0]),
to_date=all_available_dates()[-1])
return make_portfolio_performance_dataframe(stocks, timeframe,
purchases)
ld = LazyDictionary()
ld["df"] = lambda ld: data_factory(ld)
return (
cache_plot(overall_key, plot_overall_portfolio, datasets=ld),
cache_plot(stock_key, plot_portfolio_stock_performance, datasets=ld),
cache_plot(contributors_key, plot_portfolio_contributors, datasets=ld),
)
def show_total_earnings(request):
validate_user(request.user)
def data_factory(df: pd.DataFrame) -> pd.DataFrame:
df = df.pivot(
index=["asx_code", "fetch_date"], columns="field_name", values="field_value"
)
required = (df.number_of_shares > 0) & (df.eps > 0.0)
df = df[required] # ignore stocks which have unknowns
# print(df)
df["total_earnings"] = df["eps"] * df["number_of_shares"]
df = df.dropna(how="any", axis=0)
df = df.reset_index()
df = df.pivot(index="asx_code", columns="fetch_date", values="total_earnings")
df = df.merge(stocks_by_sector(), left_index=True, right_on="asx_code")
df = df.set_index("asx_code", drop=True)
df = df.groupby("sector_name").sum()
df["sector_name"] = df.index
df = df.melt(id_vars="sector_name", var_name="fetch_date")
assert set(df.columns) == set(["sector_name", "fetch_date", "value"])
df["fetch_date"] = pd.to_datetime(df["fetch_date"], format="%Y-%m-%d")
return df
def plot(df: pd.DataFrame) -> p9.ggplot:
plot = (
p9.ggplot(
df,
p9.aes(
x="fetch_date",
y="value",
color="sector_name", # group="sector_name"
),
)
+ p9.geom_line(size=1.2)
+ p9.facet_wrap("~sector_name", ncol=2, scales="free_y")
+ p9.scale_y_continuous(labels=label_shorten)
)
return user_theme(
plot,
y_axis_label="Total sector earnings ($AUD, positive contributions only)",
figure_size=(12, 14),
subplots_adjust={"wspace": 0.25},
)
ld = LazyDictionary()
ld["timeframe"] = Timeframe(past_n_days=180)
ld["pe_trends_df"] = lambda ld: pe_trends_df(ld["timeframe"])
ld["df"] = lambda ld: data_factory(ld["pe_trends_df"])
context = {
"title": "Earnings per sector over time",
"timeframe": ld["timeframe"],
"plot_uri": cache_plot(
f"total-earnings-by-sector:{ld['timeframe'].description}",
lambda ld: plot(ld["df"]),
datasets=ld,
),
}
return render(request, "total_earnings_by_sector.html", context=context)
def recalc_queryset(self, **kwargs):
if kwargs == {} or not any(
["name" in kwargs, "activity" in kwargs, "sector" in kwargs]):
return Quotation.objects.none()
wanted_name = kwargs.get("name", "")
wanted_activity = kwargs.get("activity", "")
if len(wanted_name) > 0 or len(wanted_activity) > 0:
matching_companies = find_named_companies(wanted_name,
wanted_activity)
else:
matching_companies = all_stocks()
sector = kwargs.get("sector", self.DEFAULT_SECTOR)
sector_id = int(Sector.objects.get(sector_name=sector).sector_id)
sector_stocks = all_sector_stocks(sector)
if kwargs.get("sector_enabled", False):
matching_companies = matching_companies.intersection(sector_stocks)
print("Found {} companies matching: name={} or activity={}".format(
len(matching_companies), wanted_name, wanted_activity))
report_top_n = kwargs.get("report_top_n", None)
report_bottom_n = kwargs.get("report_bottom_n", None)
self.timeframe = Timeframe(past_n_days=90)
ld = LazyDictionary()
ld["sector"] = sector
ld["sector_id"] = sector_id
ld["sector_companies"] = sector_stocks
if len(matching_companies) > 0:
ld["cip_df"] = selected_cached_stocks_cip(matching_companies,
self.timeframe)
else:
ld["cip_df"] = pd.DataFrame()
ld["sector_performance_df"] = lambda ld: make_sector_performance_dataframe(
ld["cip_df"], ld["sector_companies"])
ld["sector_performance_plot"] = lambda ld: self.sector_performance(ld)
self.ld = ld
wanted_stocks = self.filter_top_bottom(ld, matching_companies,
report_top_n, report_bottom_n)
print("Requesting valid quotes for {} stocks".format(
len(wanted_stocks)))
quotations_as_at, actual_mrd = valid_quotes_only(
"latest", ensure_date_has_data=True)
ret = quotations_as_at.filter(asx_code__in=wanted_stocks)
if len(ret) < len(wanted_stocks):
got = set([q.asx_code
for q in self.qs.all()]) if self.qs else set()
missing_stocks = wanted_stocks.difference(got)
warning(
self.request,
f"could not obtain quotes for all stocks as at {actual_mrd}: {missing_stocks}",
)
print("Showing results for {} companies".format(
len(matching_companies)))
ret, _ = latest_quote(tuple(matching_companies))
return ret
def sector_performance(self, ld: LazyDictionary) -> str:
sector = ld.get("sector")
if len(ld['sector_performance_df']) < 1:
return None
return cache_plot(
f"{sector}-sector-performance",
lambda ld: plot_sector_performance(ld["sector_performance_df"],
sector),
datasets=ld,
dont_cache=True,
)
def form_valid(self, form):
sector = form.cleaned_data.get("sector", "Communication Services")
norm_method = form.cleaned_data.get("normalisation_method", None)
n_days = form.cleaned_data.get("n_days", 30)
ld = LazyDictionary()
ld["stocks"] = lambda ld: all_sector_stocks(sector)
ld["timeframe"] = Timeframe(past_n_days=n_days)
ld["cip_df"] = lambda ld: selected_cached_stocks_cip(
ld["stocks"], ld["timeframe"])
context = self.get_context_data()
def winner_results(df: pd.DataFrame) -> list:
# compute star performers: those who are above the mean on a given day counted over all days
count = defaultdict(int)
avg = df.mean(axis=0)
for col in df.columns:
winners = df[df[col] > avg[col]][col]
for winner in winners.index:
count[winner] += 1
results = []
for asx_code, n_wins in count.items():
x = df.loc[asx_code].sum()
# avoid "dead cat bounce" stocks which fall spectacularly and then post major increases in percentage terms
if x > 0.0:
results.append((asx_code, n_wins, x))
return list(reversed(sorted(results, key=lambda t: t[2])))
context.update({
"title":
"Past {} day sector performance: box plot trends".format(n_days),
"n_days":
n_days,
"sector":
sector,
"plot_uri":
cache_plot(
f"{sector}-recent-sector-view-{ld['timeframe'].description}-{norm_method}",
lambda ld: plot_boxplot_series(
ld["cip_df"], normalisation_method=norm_method),
datasets=ld,
),
"winning_stocks":
winner_results(ld["cip_df"]),
})
return render(self.request, self.template_name, context)
import pymisca.ext as pyext
pd = pyext.pd
np = pyext.np
import pymisca.vis_util as pyvis
plt = pyvis.plt
import path, sys, os
import synotil.util as sutil
import pymisca.util as pyutil
from lazydict import LazyDictionary
template = job = LazyDictionary()
__FILE__ = path.Path(__file__)
DEPENDS = {
'meta_chip': (pyext.readData, '/home/feng/meta/meta_chip.tsv'),
'fig_meta': (pyext.readData, __FILE__.dirname() / '0726-figure-meta.tsv'),
'chipseq_targets_peaks_file':
(None, '/home/feng/static/lists/1112__ELF3__chipTarg.narrowPeak'),
}
for k in DEPENDS:
v = DEPENDS[k]
assert pyext.file__notEmpty(v[1]), (v[1], )
if v[0] is not None:
job[k] = v[0](v[1])
else:
job[k] = v[1]
@pyext.setItem(job, "chipseq_targets_peaks")
from lazydict import LazyDictionary
import pymisca.ext as pyext
obj_chipseq_db = job = LazyDictionary()
job['TYPE'] = 'CHIPSEQ_DB'
job['SOURCE_DF'] = lambda s, k: [[][0], 'must specify SOURCE_DF']
@pyext.setItem(job, '_SOURCE_DF')
def _func(job, key, SOURCE_DF):
res = SOURCE_DF.copy()
res.columns = res.columns.str.replace('[^a-zA-Z0-9_]', '_').str.upper()
assert "DATA_ACC" in res.columns, (res.columns, )
return res
@pyext.setItem(job, "init")
def _func(job, key, _SOURCE_DF):
# assert "DATA_ACC" in SOURCE_DF.columns,(SOURCE_DF.columns)
@pyext.setItem(job)
@pyext.PlainFunction
def get__chipseq__bwfile(DATA_ACC):
return get__chipseq__meta(DATA_ACC)['RPKMFILE']
@pyext.setItem(job)
@pyext.PlainFunction
def get__chipseq__meta(DATA_ACC, SOURCE_DF=_SOURCE_DF):
res = SOURCE_DF.set_index('DATA_ACC').loc[DATA_ACC].to_dict()
return res
class CompanySearch(DividendYieldSearch):
DEFAULT_SECTOR = "Communication Services"
ld = LazyDictionary()
form_class = CompanySearchForm
action_url = "/search/by-company"
def additional_context(self, context):
ld = self.ld
sector = ld.get("sector", self.DEFAULT_SECTOR
) # default to Comms. Services if not specified'
sector_id = ld.get("sector_id", None)
return {
"title": "Find by company details",
"sentiment_heatmap_title": "Heatmap for matching stocks",
# to highlight top10/bottom10 bookmarks correctly
"sector_name": sector,
"sector_id": sector_id,
"sentiment_heatmap_title": "{} sector sentiment".format(sector),
"sector_performance_plot_uri": ld.get("sector_performance_plot",
None),
"timeframe_end_performance": timeframe_end_performance(ld),
}
def sector_performance(self, ld: LazyDictionary) -> str:
sector = ld.get("sector")
if len(ld['sector_performance_df']) < 1:
return None
return cache_plot(
f"{sector}-sector-performance",
lambda ld: plot_sector_performance(ld["sector_performance_df"],
sector),
datasets=ld,
dont_cache=True,
)
def filter_top_bottom(self, ld: LazyDictionary, wanted_stocks,
report_top_n, report_bottom_n) -> set:
if report_top_n is not None or report_bottom_n is not None:
cip_sum = self.ld["cip_df"].transpose().sum().to_frame(
name="percent_cip")
# print(cip_sum)
top_N = (set(cip_sum.nlargest(report_top_n, "percent_cip").index)
if report_top_n is not None else set())
bottom_N = (set(
cip_sum.nsmallest(report_bottom_n, "percent_cip").index)
if report_bottom_n is not None else set())
wanted_stocks = top_N.union(bottom_N)
else:
return wanted_stocks
def recalc_queryset(self, **kwargs):
if kwargs == {} or not any(
["name" in kwargs, "activity" in kwargs, "sector" in kwargs]):
return Quotation.objects.none()
wanted_name = kwargs.get("name", "")
wanted_activity = kwargs.get("activity", "")
if len(wanted_name) > 0 or len(wanted_activity) > 0:
matching_companies = find_named_companies(wanted_name,
wanted_activity)
else:
matching_companies = all_stocks()
sector = kwargs.get("sector", self.DEFAULT_SECTOR)
sector_id = int(Sector.objects.get(sector_name=sector).sector_id)
sector_stocks = all_sector_stocks(sector)
if kwargs.get("sector_enabled", False):
matching_companies = matching_companies.intersection(sector_stocks)
print("Found {} companies matching: name={} or activity={}".format(
len(matching_companies), wanted_name, wanted_activity))
report_top_n = kwargs.get("report_top_n", None)
report_bottom_n = kwargs.get("report_bottom_n", None)
self.timeframe = Timeframe(past_n_days=90)
ld = LazyDictionary()
ld["sector"] = sector
ld["sector_id"] = sector_id
ld["sector_companies"] = sector_stocks
if len(matching_companies) > 0:
ld["cip_df"] = selected_cached_stocks_cip(matching_companies,
self.timeframe)
else:
ld["cip_df"] = pd.DataFrame()
ld["sector_performance_df"] = lambda ld: make_sector_performance_dataframe(
ld["cip_df"], ld["sector_companies"])
ld["sector_performance_plot"] = lambda ld: self.sector_performance(ld)
self.ld = ld
wanted_stocks = self.filter_top_bottom(ld, matching_companies,
report_top_n, report_bottom_n)
print("Requesting valid quotes for {} stocks".format(
len(wanted_stocks)))
quotations_as_at, actual_mrd = valid_quotes_only(
"latest", ensure_date_has_data=True)
ret = quotations_as_at.filter(asx_code__in=wanted_stocks)
if len(ret) < len(wanted_stocks):
got = set([q.asx_code
for q in self.qs.all()]) if self.qs else set()
missing_stocks = wanted_stocks.difference(got)
warning(
self.request,
f"could not obtain quotes for all stocks as at {actual_mrd}: {missing_stocks}",
)
print("Showing results for {} companies".format(
len(matching_companies)))
ret, _ = latest_quote(tuple(matching_companies))
return ret
def show_companies(
matching_companies, # may be QuerySet or iterable of stock codes (str)
request,
sentiment_timeframe: Timeframe,
extra_context=None,
template_name="all_stocks.html",
):
"""
Support function to public-facing views to eliminate code redundancy
"""
if isinstance(matching_companies, QuerySet):
stocks_queryset = matching_companies # we assume QuerySet is already sorted by desired criteria
elif matching_companies is None or len(matching_companies) > 0:
stocks_queryset, _ = latest_quote(matching_companies)
# FALLTHRU
else:
# no companies to report?
warning(request, "No matching companies.")
return render(request,
template_name,
context={"timeframe": sentiment_timeframe})
# prune companies without a latest price, makes no sense to report them
stocks_queryset = stocks_queryset.exclude(last_price__isnull=True)
# sort queryset as this will often be requested by the USER
arg = request.GET.get("sort_by", "asx_code")
#info(request, "Sorting by {}".format(arg))
if arg == "sector" or arg == "sector,-eps":
ss = {
s["asx_code"]: s["sector_name"]
for s in stocks_by_sector().to_dict("records")
}
if arg == "sector":
stocks_queryset = sorted(stocks_queryset,
key=lambda s: ss.get(s.asx_code, "Z")
) # companies without sector sort last
else:
eps_dict = {
s.asx_code: s.eps if s.eps is not None else 0.0
for s in stocks_queryset
}
stocks_queryset = sorted(
stocks_queryset,
key=lambda s:
(ss.get(s.asx_code, "Z"), -eps_dict.get(s.asx_code, 0.0)),
)
else:
sort_by = tuple(arg.split(","))
stocks_queryset = stocks_queryset.order_by(*sort_by)
# keep track of stock codes for template convenience
asx_codes = [quote.asx_code for quote in stocks_queryset]
n_top_bottom = (extra_context["n_top_bottom"]
if "n_top_bottom" in extra_context else 20)
print("show_companies: found {} stocks".format(len(asx_codes)))
# setup context dict for the render
context = {
# NB: title and heatmap_title are expected to be supplied by caller via extra_context
"timeframe": sentiment_timeframe,
"title": "Caller must override",
"watched": user_watchlist(request.user),
"n_stocks": len(asx_codes),
"n_top_bottom": n_top_bottom,
"virtual_purchases": user_purchases(request.user),
}
# since we sort above, we must setup the pagination also...
# assert isinstance(stocks_queryset, QuerySet)
paginator = Paginator(stocks_queryset, 50)
page_number = request.GET.get("page", 1)
page_obj = paginator.page(page_number)
context["page_obj"] = page_obj
context["object_list"] = paginator
# compute totals across all dates for the specified companies to look at top10/bottom10 in the timeframe
ld = LazyDictionary()
ld["cip_df"] = lambda ld: selected_cached_stocks_cip(
asx_codes, sentiment_timeframe)
ld["sum_by_company"] = lambda ld: ld["cip_df"].sum(axis=1,
numeric_only=True)
ld["top10"] = lambda ld: ld["sum_by_company"].nlargest(n_top_bottom)
ld["bottom10"] = lambda ld: ld["sum_by_company"].nsmallest(n_top_bottom)
ld["stocks_by_sector"] = lambda ld: stocks_by_sector()
if len(asx_codes) <= 0 or len(ld["top10"]) <= 0:
warning(request, "No matching companies found.")
else:
sorted_codes = "-".join(sorted(asx_codes))
sentiment_heatmap_uri = cache_plot(
f"{sorted_codes}-{sentiment_timeframe.description}-stocks-sentiment-plot",
lambda ld: plot_heatmap(sentiment_timeframe, ld),
datasets=ld,
)
key = f"{sorted_codes}-{sentiment_timeframe.description}-breakdown-plot"
sector_breakdown_uri = cache_plot(key, plot_breakdown, datasets=ld)
top10_plot_uri = cache_plot(
f"top10-plot-{'-'.join(ld['top10'].index)}",
lambda ld: plot_cumulative_returns(ld["top10"].index, ld),
datasets=ld,
)
bottom10_plot_uri = cache_plot(
f"bottom10-plot-{'-'.join(ld['bottom10'].index)}",
lambda ld: plot_cumulative_returns(ld["bottom10"].index, ld),
datasets=ld,
)
context.update({
"best_ten":
ld["top10"],
"worst_ten":
ld["bottom10"],
"sentiment_heatmap_uri":
sentiment_heatmap_uri,
"sentiment_heatmap_title":
"{}: {}".format(context["title"], sentiment_timeframe.description),
"sector_breakdown_uri":
sector_breakdown_uri,
"top10_plot_uri":
top10_plot_uri,
"bottom10_plot_uri":
bottom10_plot_uri,
"timeframe_end_performance":
timeframe_end_performance(ld),
})
if extra_context:
context.update(extra_context)
add_messages(request, context)
# print(context)
return render(request, template_name, context=context)
def optimise_portfolio(
stocks,
timeframe: Timeframe,
algo="ef-minvol",
max_stocks=80,
total_portfolio_value=100 * 1000,
exclude_price=None,
warning_cb=None,
**kwargs,
):
assert len(stocks) >= 1
assert timeframe is not None
assert total_portfolio_value > 0
assert max_stocks >= 5
(
all_returns,
stock_prices,
latest_prices,
first_prices,
) = setup_optimisation_matrices(stocks, timeframe, exclude_price,
warning_cb)
market_prices = company_prices(("A200", ),
Timeframe(past_n_days=180),
missing_cb=None)
market_prices.index = pd.to_datetime(market_prices.index,
format="%Y-%m-%d")
market_prices = pd.Series(market_prices["A200"])
quotes, ymd = valid_quotes_only("latest", ensure_date_has_data=True)
for t in ((10, 0.0001), (20, 0.0005), (30, 0.001), (40, 0.005), (50,
0.01)):
filtered_stocks, n_stocks = select_suitable_stocks(
all_returns, stock_prices, max_stocks, *t)
# since the sample of stocks might be different, we must recompute each iteration...
filtered_stocks = filtered_stocks.sample(n=n_stocks, axis=1)
# print(len(filtered_stocks.columns))
market_caps = {
q.asx_code: q.market_cap
for q in quotes if q.asx_code in filtered_stocks.columns
}
ld = (LazyDictionary()
) # must start a new dict since each key is immutable after use
ld["n_stocks"] = n_stocks
ld["filtered_stocks"] = filtered_stocks
ld["market_prices"] = market_prices
ld["market_caps"] = market_caps
ld["total_portfolio_value"] = total_portfolio_value
ld["returns_by"] = kwargs.get("returns_by", "by_prices")
strategy, kwargs = assign_strategy(ld, algo)
try:
run_iteration(
ld,
strategy,
first_prices,
latest_prices,
filtered_stocks,
**kwargs,
)
# NB: we dont bother caching these plots since we must calculate so many other values but we need to serve them via cache_plot() anyway
ld["efficient_frontier_plot"] = cache_plot(
secrets.token_urlsafe(32), plot_random_portfolios, datasets=ld)
ld["correlation_plot"] = lambda ld: cache_plot(
secrets.token_urlsafe(32),
lambda ld: plot_covariance(ld["m"], plot_correlation=True).
figure,
datasets=ld,
)
return ld
except ValueError as ve:
if warning_cb:
warning_cb(
"Unable to optimise stocks with min_unique={} and var_min={}: n_stocks={} - {}"
.format(t[0], t[1], n_stocks, str(ve)))
# try next iteration
raise ve
print("*** WARNING: unable to optimise portolio!")
return LazyDictionary()
def show_financial_metrics(request, stock=None):
validate_user(request.user)
validate_stock(stock)
def data_factory(ld: LazyDictionary):
data_df = financial_metrics(stock)
if data_df is None or len(data_df) < 1:
raise Http404(f"No financial metrics available for {stock}")
return data_df
def find_linear_metrics(ld: LazyDictionary) -> Iterable[str]:
linear_metrics = calculate_trends(ld["data_df"])
good_linear_metrics = []
for k, t in linear_metrics.items():
if t[1] < 0.1:
good_linear_metrics.append(k)
return good_linear_metrics
def find_exp_metrics(ld: LazyDictionary) -> Iterable[str]:
exp_metrics = calculate_trends(
ld["data_df"], polynomial_degree=2, nrmse_cutoff=0.05
)
good_linear_metrics = set(ld["linear_metrics"])
good_exp_metrics = []
for k, t in exp_metrics.items():
if t[1] < 0.1 and k not in good_linear_metrics:
good_exp_metrics.append(k)
return good_exp_metrics
ld = LazyDictionary()
ld["data_df"] = lambda ld: data_factory(ld)
ld["linear_metrics"] = lambda ld: find_linear_metrics(ld)
ld["exp_metrics"] = lambda ld: find_exp_metrics(ld)
# print(
# f"n_metrics == {len(data_df)} n_trending={len(linear_metrics.keys())} n_good_fit={len(good_linear_metrics)} n_good_exp={len(good_exp_metrics)}"
# )
def plot_metrics(df: pd.DataFrame, use_short_labels=False, **kwargs):
plot = (
p9.ggplot(df, p9.aes(x="date", y="value", colour="metric"))
+ p9.geom_line(size=1.3)
+ p9.geom_point(size=3)
)
if use_short_labels:
plot += p9.scale_y_continuous(labels=label_shorten)
n_metrics = df["metric"].nunique()
return user_theme(
plot,
subplots_adjust={"left": 0.2},
figure_size=(12, int(n_metrics * 1.5)),
**kwargs,
)
def plot_linear_trending_metrics(ld: LazyDictionary):
df = ld["data_df"].filter(ld["linear_metrics"], axis=0)
if len(df) < 1:
return None
df["metric"] = df.index
df = df.melt(id_vars="metric").dropna(how="any", axis=0)
plot = plot_metrics(df, use_short_labels=True)
plot += p9.facet_wrap("~metric", ncol=1, scales="free_y")
return plot
def plot_exponential_growth_metrics(ld: LazyDictionary):
df = ld["data_df"].filter(ld["exp_metrics"], axis=0)
if len(df) < 1:
return None
df["metric"] = df.index
df = df.melt(id_vars="metric").dropna(how="any", axis=0)
plot = plot_metrics(df)
plot += p9.facet_wrap("~metric", ncol=1, scales="free_y")
return plot
def plot_earnings_and_revenue(ld: LazyDictionary):
df = ld["data_df"].filter(["Ebit", "Total Revenue", "Earnings"], axis=0)
if len(df) < 2:
print(f"WARNING: revenue and earnings not availabe for {stock}")
return None
df["metric"] = df.index
df = df.melt(id_vars="metric").dropna(how="any", axis=0)
plot = plot_metrics(
df,
use_short_labels=True,
legend_position="right",
y_axis_label="$ AUD",
) # need to show metric name somewhere on plot
return plot
er_uri = cache_plot(
f"{stock}-earnings-revenue-plot",
lambda ld: plot_earnings_and_revenue(ld),
datasets=ld,
)
trending_metrics_uri = cache_plot(
f"{stock}-trending-metrics-plot",
lambda ld: plot_linear_trending_metrics(ld),
datasets=ld,
)
exp_growth_metrics_uri = cache_plot(
f"{stock}-exponential-growth-metrics-plot",
lambda ld: plot_exponential_growth_metrics(ld),
datasets=ld,
)
warning(
request,
"Due to experimental data ingest - data on this page may be wrong/misleading/inaccurate/missing. Use at own risk.",
)
context = {
"asx_code": stock,
"data": ld["data_df"],
"earnings_and_revenue_plot_uri": er_uri,
"trending_metrics_plot_uri": trending_metrics_uri,
"exp_growth_metrics_plot_uri": exp_growth_metrics_uri,
}
return render(request, "stock_financial_metrics.html", context=context)
def show_stock(request, stock=None, n_days=2 * 365):
"""
Displays a view of a single stock via the template and associated state
"""
validate_stock(stock)
validate_user(request.user)
plot_timeframe = Timeframe(past_n_days=n_days) # for template
def dataframe(ld: LazyDictionary) -> pd.DataFrame:
momentum_timeframe = Timeframe(
past_n_days=n_days + 200
) # to warmup MA200 function
df = company_prices(
(stock,),
momentum_timeframe,
fields=all_stock_fundamental_fields,
missing_cb=None,
)
return df
# key dynamic images and text for HTML response. We only compute the required data if image(s) not cached
# print(df)
ld = LazyDictionary()
ld["stock_df"] = lambda ld: ld["stock_df_200"].filter(
items=plot_timeframe.all_dates(), axis="rows"
)
ld["cip_df"] = lambda: cached_all_stocks_cip(plot_timeframe)
ld["stock_df_200"] = lambda ld: dataframe(ld)
ld["sector_companies"] = lambda: companies_with_same_sector(stock)
ld["company_details"] = lambda: stock_info(stock, lambda msg: warning(request, msg))
ld["sector"] = lambda ld: ld["company_details"].get("sector_name", "")
# point_score_results is a tuple (point_score_df, net_points_by_rule)
ld["point_score_results"] = lambda ld: make_point_score_dataframe(
stock, default_point_score_rules(), ld
)
ld["stock_vs_sector_df"] = lambda ld: make_stock_vs_sector_dataframe(
ld["cip_df"], stock, ld["sector_companies"]
)
print(ld["stock_vs_sector_df"])
momentum_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-rsi-plot",
lambda ld: plot_momentum(stock, plot_timeframe, ld),
datasets=ld,
)
monthly_maximum_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-monthly-maximum-plot",
lambda ld: plot_trend("M", ld),
datasets=ld,
)
monthly_returns_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-monthly returns",
lambda ld: plot_monthly_returns(plot_timeframe, stock, ld),
datasets=ld,
)
company_versus_sector_plot = cache_plot(
f"{stock}-{ld['sector']}-company-versus-sector",
lambda ld: plot_company_versus_sector(
ld["stock_vs_sector_df"], stock, ld["sector"]
),
datasets=ld,
)
point_score_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-point-score-plot",
lambda ld: plot_series(ld["point_score_results"][0], x="date", y="points"),
datasets=ld,
)
net_rule_contributors_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-rules-by-points",
lambda ld: plot_points_by_rule(ld["point_score_results"][1]),
datasets=ld,
)
# populate template and render HTML page with context
context = {
"asx_code": stock,
"watched": user_watchlist(request.user),
"timeframe": plot_timeframe,
"information": ld["company_details"],
"momentum": {
"rsi_plot": momentum_plot,
"monthly_highest_price": {
"title": "Highest price each month",
"plot_uri": monthly_maximum_plot,
},
},
"fundamentals": {
"plot_uri": cache_plot(
f"{stock}-{plot_timeframe.description}-fundamentals-plot",
lambda ld: plot_fundamentals(
fundamentals_dataframe(plot_timeframe, stock, ld),
stock,
),
datasets=ld,
),
"title": "Stock fundamentals: EPS, PE, DY etc.",
"timeframe": plot_timeframe,
},
"stock_vs_sector": {
"plot_uri": company_versus_sector_plot,
"title": "Company versus sector - percentage change",
"timeframe": plot_timeframe,
},
"point_score": {
"plot_uri": point_score_plot,
"title": "Points score due to price movements",
},
"net_contributors": {
"plot_uri": net_rule_contributors_plot,
"title": "Contributions to point score by rule",
},
"month_by_month_return_uri": monthly_returns_plot,
}
return render(request, "stock_page.html", context=context)
import pymisca.ext as pyext
pd = pyext.pd
np = pyext.np
import os
SRCDIR = os.path.dirname(__file__)
from lazydict import LazyDictionary
rnaseq_figure = template = job = LazyDictionary()
# with pyext.getPathStack([]) as stack:
# ! ls -lhtr *.pk
from util import _get_file
###############################
#### Data loading #############
rnaseq = rnaseq_raw = pyext.readData(
_get_file('/home/feng/envs/Fig_POLYQ/rnaseq.pk'))
# rnaseq = rnaseq_raw = pyext.readData(pyext.f('{SRCDIR}/static.envs.Fig_POLYQ.rnaseq.pk'))
rnaseq = rnaseq.copy()
rnaseq.loc[:] = rnaseq.apply(pyext.log2p1)
job['rnaseq'] = rnaseq
mcurr0 = pyext.readData(
_get_file('/home/feng/static/results/0318-makeRNA-polyQ/mcurr0.csv'))
# mcurr0 = pyext.readData( pyext.f('{SRCDIR}/static.results.0318-makeRNA-polyQ.mcurr0.csv') )
mcurr0.columns = mcurr0.columns.str.upper()
mcurr0['DISP_NAME'] = pyext.df__format(mcurr0,
'{TEMP}-{ZTIME}-{GTYPE}-{index}')
job['datasets_meta'] = mcurr0
keyDF = pyext.readData(pyext.f('{SRCDIR}/key_ath.csv'))
def market_sentiment(request, n_days=21, n_top_bottom=20, sector_n_days=365):
validate_user(request.user)
assert n_days > 0
assert n_top_bottom > 0
def market_cap_data_factory(ld: LazyDictionary) -> pd.DataFrame:
dates = ld["sector_timeframe"].all_dates()
# print(dates)
assert len(dates) > 90
result_df = None
adjusted_dates = []
for the_date in [dates[0], dates[-1], dates[-30], dates[-90]]:
print(f"Before valid_quotes_only for {the_date}")
quotes, actual_trading_date = valid_quotes_only(
the_date, ensure_date_has_data=True)
print(f"After valid_quotes_only for {the_date}")
print(f"Before make quotes {actual_trading_date}")
print(len(quotes))
df = make_quote_df(quotes, ld["asx_codes"], actual_trading_date)
print("After make_quote_df")
result_df = df if result_df is None else result_df.append(df)
if the_date != actual_trading_date:
adjusted_dates.append(the_date)
if len(adjusted_dates) > 0:
warning(
request,
"Some dates were not trading days, adjusted: {}".format(
adjusted_dates),
)
return result_df
ld = LazyDictionary()
ld["asx_codes"] = lambda ld: all_stocks()
ld["sector_timeframe"] = lambda ld: Timeframe(past_n_days=sector_n_days)
ld["timeframe"] = lambda ld: Timeframe(past_n_days=n_days)
ld["sector_df"] = lambda ld: cached_all_stocks_cip(ld["sector_timeframe"])
ld["sector_cumsum_df"] = lambda ld: ld["sector_df"].cumsum(axis=1)
ld["cip_df"] = lambda ld: ld["sector_df"].filter(
items=ld["timeframe"].all_dates(), axis=1)
ld["market_cap_df"] = lambda ld: market_cap_data_factory(ld)
ld["stocks_by_sector"] = lambda ld: stocks_by_sector()
sentiment_plot = cache_plot(
f"market-sentiment-{ld['timeframe'].description}",
lambda ld: plot_heatmap(ld["timeframe"], ld),
datasets=ld,
)
sector_descr = ld["sector_timeframe"].description
sector_performance_plot = cache_plot(
f"sector-performance-{sector_descr}",
lambda ld: plot_market_wide_sector_performance(ld),
datasets=ld,
)
market_cap_dist_plot = cache_plot(
f"market-cap-dist-{sector_descr}",
lambda ld: plot_market_cap_distribution(ld),
datasets=ld,
)
df = ld["sector_cumsum_df"].transpose()
df.index = pd.to_datetime(df.index, format="%Y-%m-%d")
df = (df.resample("BM", ).asfreq().diff(periods=1))
ld["monthly_returns_by_stock"] = df
# print(df)
context = {
"sentiment_uri":
sentiment_plot,
"n_days":
ld["timeframe"].n_days,
"n_stocks_plotted":
len(ld["asx_codes"]),
"n_top_bottom":
n_top_bottom,
"watched":
user_watchlist(request.user),
"sector_performance_uri":
sector_performance_plot,
"sector_timeframe":
ld["sector_timeframe"],
"sector_performance_title":
"Cumulative sector avg. performance: {}".format(
ld["sector_timeframe"].description),
"title":
"Market sentiment",
"market_cap_distribution_uri":
market_cap_dist_plot,
"monthly_sector_mean_returns":
plot_sector_monthly_mean_returns(ld),
}
return render(request, "market_sentiment_view.html", context=context)
def show_pe_trends(request):
"""
Display a plot of per-sector PE trends across stocks in each sector
ref: https://www.commsec.com.au/education/learn/choosing-investments/what-is-price-to-earnings-pe-ratio.html
"""
validate_user(request.user)
def make_pe_trends_market_avg_df(ld: LazyDictionary) -> pd.DataFrame:
df = ld["data_df"]
ss = ld["stocks_by_sector"]
pe_pos_df, _ = make_pe_trends_positive_pe_df(df, ss)
market_avg_pe_df = pe_pos_df.mean(axis=0, numeric_only=True).to_frame(
name="market_pe") # avg P/E by date series
market_avg_pe_df["date"] = pd.to_datetime(market_avg_pe_df.index)
return market_avg_pe_df
def sector_eps_data_factory(ld: LazyDictionary) -> pd.DataFrame:
df = ld["data_df"]
n_stocks = df["asx_code"].nunique()
pe_df, positive_pe_stocks = ld["positive_pe_tuple"]
eps_df = ld["eps_df"]
ss = ld["stocks_by_sector"]
# print(positive_pe_stocks)
eps_stocks = set(eps_df.index)
ss_dict = {row.asx_code: row.sector_name for row in ss.itertuples()}
# print(ss_dict)
trading_dates = set(pe_df.columns)
trading_dates.remove("sector_name")
sector_counts_all_stocks = ss["sector_name"].value_counts()
all_sectors = set(ss["sector_name"].unique())
breakdown_by_sector_pe_pos_stocks_only = pe_df[
"sector_name"].value_counts()
# print(breakdown_by_sector_pe_pos_stocks_only)
sector_counts_pe_pos_stocks_only = {
s[0]: s[1]
for s in breakdown_by_sector_pe_pos_stocks_only.items()
}
# print(sector_counts_pe_pos_stocks_only)
# print(sector_counts_all_stocks)
# print(sector_counts_pe_pos_stocks_only)
records = []
for ymd in filter(
lambda d: d in trading_dates, ld["timeframe"].all_dates()
): # needed to avoid KeyError raised during DataFrame.at[] calls below
sum_pe_per_sector = defaultdict(float)
sum_eps_per_sector = defaultdict(float)
for stock in filter(lambda code: code in ss_dict,
positive_pe_stocks):
sector = ss_dict[stock]
assert isinstance(sector, str)
if stock in eps_stocks:
eps = eps_df.at[stock, ymd]
if isnan(eps):
continue
sum_eps_per_sector[sector] += eps
if stock in positive_pe_stocks:
pe = pe_df.at[stock, ymd]
if isnan(pe):
continue
assert pe >= 0.0
sum_pe_per_sector[sector] += pe
# print(len(sector_counts_all_stocks))
# print(len(sum_eps_per_sector))
assert len(sector_counts_pe_pos_stocks_only) >= len(
sum_pe_per_sector)
assert len(sector_counts_all_stocks) >= len(sum_eps_per_sector)
for sector in all_sectors:
pe_sum = sum_pe_per_sector.get(sector, None)
n_pe = sector_counts_pe_pos_stocks_only.get(sector, None)
pe_mean = pe_sum / n_pe if pe_sum is not None else None
eps_sum = sum_eps_per_sector.get(sector, None)
records.append({
"date": ymd,
"sector": sector,
"mean_pe": pe_mean,
"sum_pe": pe_sum,
"sum_eps": eps_sum,
"n_stocks": n_stocks,
"n_sector_stocks_pe_only": n_pe,
})
df = pd.DataFrame.from_records(records)
# print(df[df["sector"] == 'Utilities'])
# print(df)
return df
ld = LazyDictionary()
ld["data_df"] = lambda ld: pe_trends_df(ld["timeframe"])
ld["positive_pe_tuple"] = lambda ld: make_pe_trends_positive_pe_df(
ld["data_df"], ld["stocks_by_sector"])
ld["market_avg_pe_df"] = lambda ld: make_pe_trends_market_avg_df(ld)
ld["eps_df"] = lambda ld: make_pe_trends_eps_df(ld["data_df"])
ld["sector_eps_df"] = lambda ld: sector_eps_data_factory(ld)
ld["stocks_by_sector"] = stocks_by_sector()
ld["timeframe"] = Timeframe(past_n_days=180)
td = ld["timeframe"].description
# these arent per-user plots: they can safely be shared across all users of the site, so the key reflects that
sector_pe_cache_key = f"{td}-by-sector-pe-plot"
sector_eps_cache_key = f"{td}-by-sector-eps-plot"
market_pe_cache_key = f"{td}-market-pe-mean"
market_pe_plot_uri = cache_plot(
market_pe_cache_key,
lambda ld: plot_series(
ld["market_avg_pe_df"],
x="date",
y="market_pe",
y_axis_label="Market-wide mean P/E",
color=None,
use_smooth_line=True,
),
datasets=ld,
)
context = {
"title":
"PE Trends",
"n_stocks":
ld["data_df"]["asx_code"].nunique(),
"timeframe":
ld["timeframe"],
"n_stocks_with_pe":
len(ld["positive_pe_tuple"][1]),
"sector_pe_plot_uri":
cache_plot(
sector_pe_cache_key,
lambda ld: plot_sector_field(ld["sector_eps_df"], field="mean_pe"),
datasets=ld,
),
"sector_eps_plot_uri":
cache_plot(
sector_eps_cache_key,
lambda ld: plot_sector_field(ld["sector_eps_df"], field="sum_eps"),
datasets=ld,
),
"market_pe_plot_uri":
market_pe_plot_uri,
"sector_positive_top_contributors_eps_uri":
cache_plot(
f"top-contributors-{sector_eps_cache_key}",
lambda ld: plot_sector_top_eps_contributors(
ld["eps_df"], ld["stocks_by_sector"]),
datasets=ld,
),
}
return render(request, "pe_trends.html", context)