def render_to_response(self, context):
qs = context['paginator'].object_list.all(
) # all() to get a fresh queryset instance
qs = list(qs.values_list('asx_code', flat=True))
if len(qs) == 0:
warning(self.request, "No stocks to report")
sentiment_data, df, top10, bottom10, n_stocks = (None, None, None,
None, 0)
else:
sentiment_data, df, top10, bottom10, n_stocks = plot_heatmap(
qs, n_top_bottom=self.n_top_bottom)
context.update({
'most_recent_date':
self.as_at_date,
'sentiment_heatmap':
sentiment_data,
'watched':
user_watchlist(
self.request.user), # to ensure bookmarks are correct
'n_top_bottom':
self.n_top_bottom,
'best_ten':
top10,
'worst_ten':
bottom10,
'title':
'Find by dividend yield or P/E',
'sentiment_heatmap_title':
"Recent sentiment: {} total stocks".format(n_stocks),
})
add_messages(self.request, context)
return super().render_to_response(context)
def get_queryset(self, **kwargs):
# user never run this view before?
if kwargs == {}:
print("WARNING: no form parameters specified - returning empty queryset")
return Quotation.objects.none()
self.sector = kwargs.get("sector", self.sector)
self.sector_id = int(Sector.objects.get(sector_name=self.sector).sector_id)
wanted_stocks = all_sector_stocks(self.sector)
print("Found {} stocks matching sector={}".format(len(wanted_stocks), self.sector))
mrd = latest_quotation_date('ANZ')
report_top_n = kwargs.get('report_top_n', None)
report_bottom_n = kwargs.get('report_bottom_n', None)
if report_top_n is not None or report_bottom_n is not None:
cip_sum = selected_cached_stocks_cip(wanted_stocks, Timeframe(past_n_days=90)).transpose().sum().to_frame(name="percent_cip")
#print(cip_sum)
top_N = set(cip_sum.nlargest(report_top_n, "percent_cip").index) if report_top_n is not None else set()
bottom_N = set(cip_sum.nsmallest(report_bottom_n, "percent_cip").index) if report_bottom_n is not None else set()
wanted_stocks = top_N.union(bottom_N)
print("Requesting valid quotes for {} stocks".format(len(wanted_stocks)))
self.qs = valid_quotes_only(mrd).filter(asx_code__in=wanted_stocks)
if len(self.qs) < len(wanted_stocks):
got = set([q.asx_code for q in self.qs.all()])
missing_stocks = wanted_stocks.difference(got)
warning(self.request, f"could not obtain quotes for all stocks as at {mrd}: {missing_stocks}")
return self.qs
def market_cap_data_factory(ld: LazyDictionary) -> pd.DataFrame:
dates = ld["sector_timeframe"].all_dates()
# print(dates)
assert len(dates) > 90
result_df = None
adjusted_dates = []
for the_date in [dates[0], dates[-1], dates[-30], dates[-90]]:
print(f"Before valid_quotes_only for {the_date}")
quotes, actual_trading_date = valid_quotes_only(
the_date, ensure_date_has_data=True)
print(f"After valid_quotes_only for {the_date}")
print(f"Before make quotes {actual_trading_date}")
print(len(quotes))
df = make_quote_df(quotes, ld["asx_codes"], actual_trading_date)
print("After make_quote_df")
result_df = df if result_df is None else result_df.append(df)
if the_date != actual_trading_date:
adjusted_dates.append(the_date)
if len(adjusted_dates) > 0:
warning(
request,
"Some dates were not trading days, adjusted: {}".format(
adjusted_dates),
)
return result_df
def rule_at_end_of_daily_range(state: dict):
"""
Award 1 point if the price at the end of the day is within 20% of the daily trading range (either end)
Otherwise 0.
"""
assert state is not None
day_low_high_df = state.get('day_low_high_df')
date = state.get('date')
threshold = state.get('daily_range_threshold')
try:
day_low = day_low_high_df.at[date, 'day_low_price']
day_high = day_low_high_df.at[date, 'day_high_price']
last_price = day_low_high_df.at[date, 'last_price']
if np.isnan(day_low) and np.isnan(day_high):
return 0
day_range = (day_high -
day_low) * threshold # 20% at either end of daily range
if last_price >= day_high - day_range:
return 1
elif last_price <= day_low + day_range:
return -1
# else FALLTHRU...
except KeyError:
stock = state.get('stock')
warning(
None,
"Unable to obtain day low/high and last_price for {} on {}".format(
stock, date))
return 0
def recalc_queryset(self, **kwargs):
if kwargs == {} or not any(
["name" in kwargs, "activity" in kwargs, "sector" in kwargs]):
return Quotation.objects.none()
wanted_name = kwargs.get("name", "")
wanted_activity = kwargs.get("activity", "")
if len(wanted_name) > 0 or len(wanted_activity) > 0:
matching_companies = find_named_companies(wanted_name,
wanted_activity)
else:
matching_companies = all_stocks()
sector = kwargs.get("sector", self.DEFAULT_SECTOR)
sector_id = int(Sector.objects.get(sector_name=sector).sector_id)
sector_stocks = all_sector_stocks(sector)
if kwargs.get("sector_enabled", False):
matching_companies = matching_companies.intersection(sector_stocks)
print("Found {} companies matching: name={} or activity={}".format(
len(matching_companies), wanted_name, wanted_activity))
report_top_n = kwargs.get("report_top_n", None)
report_bottom_n = kwargs.get("report_bottom_n", None)
self.timeframe = Timeframe(past_n_days=90)
ld = LazyDictionary()
ld["sector"] = sector
ld["sector_id"] = sector_id
ld["sector_companies"] = sector_stocks
if len(matching_companies) > 0:
ld["cip_df"] = selected_cached_stocks_cip(matching_companies,
self.timeframe)
else:
ld["cip_df"] = pd.DataFrame()
ld["sector_performance_df"] = lambda ld: make_sector_performance_dataframe(
ld["cip_df"], ld["sector_companies"])
ld["sector_performance_plot"] = lambda ld: self.sector_performance(ld)
self.ld = ld
wanted_stocks = self.filter_top_bottom(ld, matching_companies,
report_top_n, report_bottom_n)
print("Requesting valid quotes for {} stocks".format(
len(wanted_stocks)))
quotations_as_at, actual_mrd = valid_quotes_only(
"latest", ensure_date_has_data=True)
ret = quotations_as_at.filter(asx_code__in=wanted_stocks)
if len(ret) < len(wanted_stocks):
got = set([q.asx_code
for q in self.qs.all()]) if self.qs else set()
missing_stocks = wanted_stocks.difference(got)
warning(
self.request,
f"could not obtain quotes for all stocks as at {actual_mrd}: {missing_stocks}",
)
print("Showing results for {} companies".format(
len(matching_companies)))
ret, _ = latest_quote(tuple(matching_companies))
return ret
def detect_outliers(stocks: list, all_stocks_cip: pd.DataFrame, rules=None):
"""
Returns a dataframe describing those outliers present in stocks based on the provided rules.
"""
if rules is None:
rules = default_point_score_rules()
str_rules = { str(r):r for r in rules }
rows = []
stocks_by_sector_df = stocks_by_sector() # NB: ETFs in watchlist will have no sector
stocks_by_sector_df.index = stocks_by_sector_df['asx_code']
for stock in stocks:
#print("Processing stock: ", stock)
try:
sector = stocks_by_sector_df.at[stock, 'sector_name']
sector_companies = list(stocks_by_sector_df.loc[stocks_by_sector_df['sector_name'] == sector].asx_code)
# day_low_high() may raise KeyError when data is currently being fetched, so it appears here...
day_low_high_df = day_low_high(stock, all_stocks_cip.columns)
except KeyError:
warning(None, "Unable to locate watchlist entry: {} - continuing without it".format(stock))
continue
state = {
'day_low_high_df': day_low_high_df, # never changes each day, so we init it here
'all_stocks_change_in_percent_df': all_stocks_cip,
'stock': stock,
'daily_range_threshold': 0.20, # 20% at either end of the daily range gets a point
}
points_by_rule = defaultdict(int)
for date in all_stocks_cip.columns:
market_avg = all_stocks_cip[date].mean()
sector_avg = all_stocks_cip[date].filter(items=sector_companies).mean()
stock_move = all_stocks_cip.at[stock, date]
state.update({ 'market_avg': market_avg, 'sector_avg': sector_avg,
'stock_move': stock_move, 'date': date })
for rule_name, rule in str_rules.items():
points_by_rule[rule_name] += rule(state)
d = { 'stock': stock }
d.update(points_by_rule)
rows.append(d)
df = pd.DataFrame.from_records(rows)
df = df.set_index('stock')
print(df)
from pyod.models.iforest import IForest
clf = IForest()
clf.fit(df)
scores = clf.predict(df)
results = [row[0] for row, value in zip(df.iterrows(), scores) if value > 0]
#print(results)
print("Found {} outlier stocks".format(len(results)))
return results
def render_to_response(self, context):
context.update({
"title": "Find companies by financial metric",
"sentiment_heatmap_title": "Matching stock sentiment",
})
warning(
self.request,
"Due to experimental data ingest, results may be wrong/inaccurate/misleading. Use at own risk",
)
return show_companies(
self.object_list, # ie. return result from self.get_queryset()
self.request,
Timeframe(past_n_days=30),
context,
self.template_name,
)
def form_valid(self, form):
exclude = form.cleaned_data["excluded_stocks"]
n_days = form.cleaned_data["n_days"]
algo = form.cleaned_data["method"]
portfolio_cost = form.cleaned_data["portfolio_cost"]
exclude_price = form.cleaned_data.get("exclude_price", None)
max_stocks = form.cleaned_data.get("max_stocks", 80)
stocks = self.stocks()
if exclude is not None:
if isinstance(exclude, str):
exclude = exclude.split(",")
stocks = set(stocks).difference(exclude)
if form.cleaned_data["exclude_etfs"]:
stocks = set(stocks).difference(all_etfs())
print(f"After excluding ETFs: {len(stocks)} stocks remain")
self.timeframe = Timeframe(past_n_days=n_days)
self.results = optimise_portfolio(
stocks,
self.timeframe,
algo=algo,
max_stocks=max_stocks,
total_portfolio_value=portfolio_cost,
exclude_price=exclude_price,
warning_cb=lambda msg: warning(self.request, msg),
returns_by=form.cleaned_data.get("returns_by", None),
)
return render(self.request, self.template_name,
self.get_context_data())
def form_valid(self, form):
df = fetch_dataframe(self.data_flow.name)
if df is None or len(df) == 0:
raise Http404(f"Unable to load dataframe: {self.data_flow}")
filter_performance = []
for k, v in form.cleaned_data.items():
rows_at_start = len(df)
print(
f"Filtering rows for {k}: total {rows_at_start} rows at start")
k = k[len("dimension_"):]
if rows_at_start < 10000:
unique_values_left = df[k].unique()
else:
unique_values_left = set()
df = df[df[k] == v]
rows_at_end = len(df)
filter_performance.append(
(k, v, rows_at_start, rows_at_end, unique_values_left))
print(f"After filtering: now {rows_at_end} rows")
if len(df) == 0:
warning(self.request,
f"No rows of data left after filtering: {k} {v}")
break
plot = None
plot_title = ""
if len(df) > 0:
plot_title, x_axis_column, y_axis_column, df = detect_dataframe(
df, self.data_flow)
plot = (p9.ggplot(df, p9.aes(x=x_axis_column, y=y_axis_column)) +
p9.geom_point() + p9.geom_line())
plot = user_theme(plot)
context = self.get_context_data()
cache_key = "-".join(sorted(form.cleaned_data.values())) + "-ecb-plot"
context.update({
"dataflow": self.data_flow,
"dataflow_name": self.data_flow.name,
"filter_performance": filter_performance,
"plot_title": plot_title,
"plot_uri": cache_plot(cache_key, lambda: plot),
})
return render(self.request, self.template_name, context)
def get_initial(self, **kwargs):
stock = kwargs.get("stock", self.kwargs.get("stock"))
amount = kwargs.get("amount", self.kwargs.get("amount", 5000.0))
user = self.request.user
validate_stock(stock)
validate_user(user)
quote, latest_date = latest_quote(stock)
cur_price = quote.last_price
if cur_price >= 1e-6:
return {
"asx_code": stock,
"user": user,
"buy_date": latest_date,
"price_at_buy_date": cur_price,
"amount": amount,
"n": int(amount / cur_price),
}
else:
warning(self.request,
"Cannot buy {} as its price is zero/unknown".format(stock))
return {}
def get_initial(self, **kwargs):
stock = self.kwargs.get('stock')
amount = self.kwargs.get('amount', 5000.0)
user = self.request.user
validate_stock(stock)
validate_user(user)
quote, latest_date = latest_quote(stock)
cur_price = quote.last_price
if cur_price >= 1e-6:
return {
'asx_code': stock,
'user': user,
'buy_date': latest_date,
'price_at_buy_date': cur_price,
'amount': amount,
'n': int(amount / cur_price)
}
else:
warning(self.request,
"Cannot buy {} as its price is zero/unknown".format(stock))
return {}
def show_stock_sector(request, stock):
validate_stock(stock)
validate_user(request.user)
_, company_details = stock_info(stock, lambda msg: warning(request, msg))
sector = company_details.sector_name if company_details else None
all_stocks_cip = cached_all_stocks_cip(Timeframe(past_n_days=180))
# invoke separate function to cache the calls when we can
c_vs_s_plot, sector_momentum_plot, sector_companies = analyse_sector_performance(
stock, sector, all_stocks_cip)
point_score_plot = net_rule_contributors_plot = None
if sector_companies is not None:
point_score_plot, net_rule_contributors_plot = \
plot_point_scores(stock,
sector_companies,
all_stocks_cip,
default_point_score_rules())
context = {
"is_sector":
True,
"asx_code":
stock,
"sector_momentum_plot":
sector_momentum_plot,
"sector_momentum_title":
"{} sector stocks".format(sector),
"company_versus_sector_plot":
c_vs_s_plot,
"company_versus_sector_title":
"{} vs. {} performance".format(stock, sector),
"point_score_plot":
point_score_plot,
"point_score_plot_title":
"Points score due to price movements",
"net_contributors_plot":
net_rule_contributors_plot,
"net_contributors_plot_title":
"Contributions to point score by rule",
}
return render(request, "stock_sector.html", context)
def show_stock(request, stock=None, n_days=2 * 365):
"""
Displays a view of a single stock via the stock_view.html template and associated state
"""
validate_stock(stock)
validate_user(request.user)
timeframe = Timeframe(
past_n_days=n_days + 200
) # add 200 days so MA 200 can initialise itself before the plotting starts...
stock_df = rsi_data(
stock, timeframe) # may raise 404 if too little data available
securities, company_details = stock_info(stock,
lambda msg: warning(request, msg))
momentum_plot = make_rsi_plot(stock, stock_df)
# plot the price over timeframe in monthly blocks
prices = stock_df[[
'last_price'
]].transpose() # use list of columns to ensure pd.DataFrame not pd.Series
#print(prices)
monthly_maximum_plot = plot_trend(prices, sample_period='M')
# populate template and render HTML page with context
context = {
"asx_code": stock,
"securities": securities,
"cd": company_details,
"rsi_plot": momentum_plot,
"is_momentum": True,
"monthly_highest_price_plot_title": "Maximum price each month trend",
"monthly_highest_price_plot": monthly_maximum_plot,
"timeframe": f"{n_days} days",
"watched": user_watchlist(request.user),
}
return render(request, "stock_view.html", context=context)
def show_stock(request, stock=None, sector_n_days=90):
"""
Displays a view of a single stock via the stock_view.html template and associated state
"""
validate_stock(stock)
validate_user(request.user)
window_size = 14 # since must have a full window before computing momentum over sector_n_days
all_dates = desired_dates(start_date=sector_n_days + window_size)
wanted_fields = [
'last_price', 'volume', 'day_low_price', 'day_high_price', 'eps', 'pe',
'annual_dividend_yield'
]
stock_df = company_prices([stock],
all_dates=all_dates,
fields=wanted_fields)
#print(stock_df)
securities = Security.objects.filter(asx_code=stock)
company_details = CompanyDetails.objects.filter(asx_code=stock).first()
if company_details is None:
warning(request, "No details available for {}".format(stock))
n_dates = len(stock_df)
if n_dates < 14: # RSI requires at least 14 prices to plot so reject recently added stocks
raise Http404("Insufficient price quotes for {} - only {}".format(
stock, n_dates))
# plot relative strength
fig = make_rsi_plot(stock, stock_df)
# show sector performance over past 3 months
all_stocks_cip = company_prices(None,
all_dates=all_dates,
fields='change_in_percent',
fix_missing=False)
sector = company_details.sector_name if company_details else None
t = analyse_sector(stock, sector, all_stocks_cip, window_size=window_size)
c_vs_s_plot, sector_momentum_plot, point_score_plot = t
# key indicator performance over past 90 days (for now): pe, eps, yield etc.
key_indicator_plot = plot_key_stock_indicators(stock_df, stock)
# plot the price over last 600 days in monthly blocks ie. max 24 bars which is still readable
monthly_maximum_plot = plot_best_monthly_price_trend(
all_quotes(stock, all_dates=desired_dates(start_date=600)))
# populate template and render HTML page with context
context = {
'rsi_data':
fig,
'asx_code':
stock,
'securities':
securities,
'cd':
company_details,
'sector_momentum_plot':
sector_momentum_plot,
'sector_momentum_title':
"{} sector stocks: {} day performance".format(sector, sector_n_days),
'company_versus_sector_plot':
c_vs_s_plot,
'company_versus_sector_title':
'{} vs. {} performance'.format(stock, sector),
'key_indicators_plot':
key_indicator_plot,
'monthly_highest_price_plot_title':
'Maximum price each month trend',
'monthly_highest_price_plot':
monthly_maximum_plot,
'point_score_plot':
point_score_plot,
'point_score_plot_title':
'Points score due to price movements'
}
return render(request, "stock_view.html", context=context)
def company_prices(stock_codes,
all_dates=None,
fields='last_price',
fail_missing_months=True,
fix_missing=True):
"""
Return a dataframe with the required companies (iff quoted) over the
specified dates. By default last_price is provided. Fields may be a list,
in which case the dataframe has columns for each field and dates are rows (in this case only one stock is permitted)
"""
if not isinstance(fields, str): # assume iterable if not str...
assert len(stock_codes) == 1
dataframes = [
company_prices(stock_codes,
all_dates=all_dates,
fields=field,
fail_missing_months=fail_missing_months)
for field in fields
]
result_df = pd.concat(dataframes, ignore_index=True)
result_df.set_index(pd.Index(fields), inplace=True)
#print(result_df)
result_df = result_df.transpose()
#print(result_df)
assert list(result_df.columns) == fields
# reject rows which are all NA to avoid downstream problems eg. plotting stocks
# NB: we ONLY do this for the multi-field case, single field it is callers responsibility
result_df = result_df.dropna()
return result_df
#print(stock_codes)
assert isinstance(fields, str)
if all_dates is None:
all_dates = [datetime.strftime(datetime.now(), "%Y-%m-%d")]
required_tags = set()
for date in all_dates:
validate_date(date)
yyyy = date[0:4]
mm = date[5:7]
required_tags.add("{}-{}-{}-asx".format(fields, mm, yyyy))
which_cols = set(all_dates)
# construct a "super" dataframe from the constituent parquet data
superdf, n_dataframes = make_superdf(required_tags, stock_codes)
# drop columns not present in all_dates to ensure we are giving just the results requested
cols_to_drop = [date for date in superdf.columns if date not in which_cols]
superdf = superdf.drop(columns=cols_to_drop)
# on the first of the month, we dont have data yet so we permit one missing tag for this reason
if fail_missing_months and n_dataframes < len(required_tags) - 1:
raise ValueError(
"Not all required data is available - aborting! Found {} wanted {}"
.format(n_dataframes, required_tags))
# NB: ensure all columns are ALWAYS in ascending date order
dates = sorted(list(superdf.columns),
key=lambda k: datetime.strptime(k, "%Y-%m-%d"))
superdf = superdf[dates]
if fix_missing and superdf.isnull().values.any():
warning(
None,
"Missing data found in fields={} stocks={} over dates: {}-{}".
format(fields, stock_codes, all_dates[0], all_dates[-1]))
superdf = impute_missing(superdf)
return superdf
def show_companies(
matching_companies, # may be QuerySet or iterable of stock codes (str)
request,
sentiment_timeframe: Timeframe,
extra_context=None,
template_name="all_stocks.html",
):
"""
Support function to public-facing views to eliminate code redundancy
"""
virtual_purchases_by_user = user_purchases(request.user)
if isinstance(matching_companies, QuerySet):
stocks_queryset = matching_companies # we assume QuerySet is already sorted by desired criteria
elif matching_companies is None or len(matching_companies) > 0:
stocks_queryset, _ = latest_quote(matching_companies)
# FALLTHRU
# sort queryset as this will often be requested by the USER
sort_by = tuple(request.GET.get("sort_by", "asx_code").split(","))
info(request, "Sorting by {}".format(sort_by))
stocks_queryset = stocks_queryset.order_by(*sort_by)
# keep track of stock codes for template convenience
asx_codes = [quote.asx_code for quote in stocks_queryset.all()]
n_top_bottom = extra_context['n_top_bottom'] if 'n_top_bottom' in extra_context else 20
print("show_companies: found {} stocks".format(len(asx_codes)))
# setup context dict for the render
context = {
# NB: title and heatmap_title are expected to be supplied by caller via extra_context
"timeframe": sentiment_timeframe,
"title": "Caller must override",
"watched": user_watchlist(request.user),
"n_stocks": len(asx_codes),
"n_top_bottom": n_top_bottom,
"virtual_purchases": virtual_purchases_by_user,
}
# since we sort above, we must setup the pagination also...
assert isinstance(stocks_queryset, QuerySet)
paginator = Paginator(stocks_queryset, 50)
page_number = request.GET.get("page", 1)
page_obj = paginator.page(page_number)
context['page_obj'] = page_obj
context['object_list'] = paginator
if len(asx_codes) <= 0:
warning(request, "No matching companies found.")
else:
df = selected_cached_stocks_cip(asx_codes, sentiment_timeframe)
sentiment_heatmap_data, top10, bottom10 = plot_heatmap(df, sentiment_timeframe, n_top_bottom=n_top_bottom)
sector_breakdown_plot = plot_breakdown(df)
context.update({
"best_ten": top10,
"worst_ten": bottom10,
"sentiment_heatmap": sentiment_heatmap_data,
"sentiment_heatmap_title": "{}: {}".format(context['title'], sentiment_timeframe.description),
"sector_breakdown_plot": sector_breakdown_plot,
})
if extra_context:
context.update(extra_context)
add_messages(request, context)
#print(context)
return render(request, template_name, context=context)
def show_stock(request, stock=None, n_days=2 * 365):
"""
Displays a view of a single stock via the template and associated state
"""
validate_stock(stock)
validate_user(request.user)
plot_timeframe = Timeframe(past_n_days=n_days) # for template
def dataframe(ld: LazyDictionary) -> pd.DataFrame:
momentum_timeframe = Timeframe(
past_n_days=n_days + 200
) # to warmup MA200 function
df = company_prices(
(stock,),
momentum_timeframe,
fields=all_stock_fundamental_fields,
missing_cb=None,
)
return df
# key dynamic images and text for HTML response. We only compute the required data if image(s) not cached
# print(df)
ld = LazyDictionary()
ld["stock_df"] = lambda ld: ld["stock_df_200"].filter(
items=plot_timeframe.all_dates(), axis="rows"
)
ld["cip_df"] = lambda: cached_all_stocks_cip(plot_timeframe)
ld["stock_df_200"] = lambda ld: dataframe(ld)
ld["sector_companies"] = lambda: companies_with_same_sector(stock)
ld["company_details"] = lambda: stock_info(stock, lambda msg: warning(request, msg))
ld["sector"] = lambda ld: ld["company_details"].get("sector_name", "")
# point_score_results is a tuple (point_score_df, net_points_by_rule)
ld["point_score_results"] = lambda ld: make_point_score_dataframe(
stock, default_point_score_rules(), ld
)
ld["stock_vs_sector_df"] = lambda ld: make_stock_vs_sector_dataframe(
ld["cip_df"], stock, ld["sector_companies"]
)
print(ld["stock_vs_sector_df"])
momentum_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-rsi-plot",
lambda ld: plot_momentum(stock, plot_timeframe, ld),
datasets=ld,
)
monthly_maximum_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-monthly-maximum-plot",
lambda ld: plot_trend("M", ld),
datasets=ld,
)
monthly_returns_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-monthly returns",
lambda ld: plot_monthly_returns(plot_timeframe, stock, ld),
datasets=ld,
)
company_versus_sector_plot = cache_plot(
f"{stock}-{ld['sector']}-company-versus-sector",
lambda ld: plot_company_versus_sector(
ld["stock_vs_sector_df"], stock, ld["sector"]
),
datasets=ld,
)
point_score_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-point-score-plot",
lambda ld: plot_series(ld["point_score_results"][0], x="date", y="points"),
datasets=ld,
)
net_rule_contributors_plot = cache_plot(
f"{plot_timeframe.description}-{stock}-rules-by-points",
lambda ld: plot_points_by_rule(ld["point_score_results"][1]),
datasets=ld,
)
# populate template and render HTML page with context
context = {
"asx_code": stock,
"watched": user_watchlist(request.user),
"timeframe": plot_timeframe,
"information": ld["company_details"],
"momentum": {
"rsi_plot": momentum_plot,
"monthly_highest_price": {
"title": "Highest price each month",
"plot_uri": monthly_maximum_plot,
},
},
"fundamentals": {
"plot_uri": cache_plot(
f"{stock}-{plot_timeframe.description}-fundamentals-plot",
lambda ld: plot_fundamentals(
fundamentals_dataframe(plot_timeframe, stock, ld),
stock,
),
datasets=ld,
),
"title": "Stock fundamentals: EPS, PE, DY etc.",
"timeframe": plot_timeframe,
},
"stock_vs_sector": {
"plot_uri": company_versus_sector_plot,
"title": "Company versus sector - percentage change",
"timeframe": plot_timeframe,
},
"point_score": {
"plot_uri": point_score_plot,
"title": "Points score due to price movements",
},
"net_contributors": {
"plot_uri": net_rule_contributors_plot,
"title": "Contributions to point score by rule",
},
"month_by_month_return_uri": monthly_returns_plot,
}
return render(request, "stock_page.html", context=context)
def show_companies(
matching_companies, # may be QuerySet or iterable of stock codes (str)
request,
sentiment_timeframe: Timeframe,
extra_context=None,
template_name="all_stocks.html",
):
"""
Support function to public-facing views to eliminate code redundancy
"""
if isinstance(matching_companies, QuerySet):
stocks_queryset = matching_companies # we assume QuerySet is already sorted by desired criteria
elif matching_companies is None or len(matching_companies) > 0:
stocks_queryset, _ = latest_quote(matching_companies)
# FALLTHRU
else:
# no companies to report?
warning(request, "No matching companies.")
return render(request,
template_name,
context={"timeframe": sentiment_timeframe})
# prune companies without a latest price, makes no sense to report them
stocks_queryset = stocks_queryset.exclude(last_price__isnull=True)
# sort queryset as this will often be requested by the USER
arg = request.GET.get("sort_by", "asx_code")
#info(request, "Sorting by {}".format(arg))
if arg == "sector" or arg == "sector,-eps":
ss = {
s["asx_code"]: s["sector_name"]
for s in stocks_by_sector().to_dict("records")
}
if arg == "sector":
stocks_queryset = sorted(stocks_queryset,
key=lambda s: ss.get(s.asx_code, "Z")
) # companies without sector sort last
else:
eps_dict = {
s.asx_code: s.eps if s.eps is not None else 0.0
for s in stocks_queryset
}
stocks_queryset = sorted(
stocks_queryset,
key=lambda s:
(ss.get(s.asx_code, "Z"), -eps_dict.get(s.asx_code, 0.0)),
)
else:
sort_by = tuple(arg.split(","))
stocks_queryset = stocks_queryset.order_by(*sort_by)
# keep track of stock codes for template convenience
asx_codes = [quote.asx_code for quote in stocks_queryset]
n_top_bottom = (extra_context["n_top_bottom"]
if "n_top_bottom" in extra_context else 20)
print("show_companies: found {} stocks".format(len(asx_codes)))
# setup context dict for the render
context = {
# NB: title and heatmap_title are expected to be supplied by caller via extra_context
"timeframe": sentiment_timeframe,
"title": "Caller must override",
"watched": user_watchlist(request.user),
"n_stocks": len(asx_codes),
"n_top_bottom": n_top_bottom,
"virtual_purchases": user_purchases(request.user),
}
# since we sort above, we must setup the pagination also...
# assert isinstance(stocks_queryset, QuerySet)
paginator = Paginator(stocks_queryset, 50)
page_number = request.GET.get("page", 1)
page_obj = paginator.page(page_number)
context["page_obj"] = page_obj
context["object_list"] = paginator
# compute totals across all dates for the specified companies to look at top10/bottom10 in the timeframe
ld = LazyDictionary()
ld["cip_df"] = lambda ld: selected_cached_stocks_cip(
asx_codes, sentiment_timeframe)
ld["sum_by_company"] = lambda ld: ld["cip_df"].sum(axis=1,
numeric_only=True)
ld["top10"] = lambda ld: ld["sum_by_company"].nlargest(n_top_bottom)
ld["bottom10"] = lambda ld: ld["sum_by_company"].nsmallest(n_top_bottom)
ld["stocks_by_sector"] = lambda ld: stocks_by_sector()
if len(asx_codes) <= 0 or len(ld["top10"]) <= 0:
warning(request, "No matching companies found.")
else:
sorted_codes = "-".join(sorted(asx_codes))
sentiment_heatmap_uri = cache_plot(
f"{sorted_codes}-{sentiment_timeframe.description}-stocks-sentiment-plot",
lambda ld: plot_heatmap(sentiment_timeframe, ld),
datasets=ld,
)
key = f"{sorted_codes}-{sentiment_timeframe.description}-breakdown-plot"
sector_breakdown_uri = cache_plot(key, plot_breakdown, datasets=ld)
top10_plot_uri = cache_plot(
f"top10-plot-{'-'.join(ld['top10'].index)}",
lambda ld: plot_cumulative_returns(ld["top10"].index, ld),
datasets=ld,
)
bottom10_plot_uri = cache_plot(
f"bottom10-plot-{'-'.join(ld['bottom10'].index)}",
lambda ld: plot_cumulative_returns(ld["bottom10"].index, ld),
datasets=ld,
)
context.update({
"best_ten":
ld["top10"],
"worst_ten":
ld["bottom10"],
"sentiment_heatmap_uri":
sentiment_heatmap_uri,
"sentiment_heatmap_title":
"{}: {}".format(context["title"], sentiment_timeframe.description),
"sector_breakdown_uri":
sector_breakdown_uri,
"top10_plot_uri":
top10_plot_uri,
"bottom10_plot_uri":
bottom10_plot_uri,
"timeframe_end_performance":
timeframe_end_performance(ld),
})
if extra_context:
context.update(extra_context)
add_messages(request, context)
# print(context)
return render(request, template_name, context=context)
def show_financial_metrics(request, stock=None):
validate_user(request.user)
validate_stock(stock)
def data_factory(ld: LazyDictionary):
data_df = financial_metrics(stock)
if data_df is None or len(data_df) < 1:
raise Http404(f"No financial metrics available for {stock}")
return data_df
def find_linear_metrics(ld: LazyDictionary) -> Iterable[str]:
linear_metrics = calculate_trends(ld["data_df"])
good_linear_metrics = []
for k, t in linear_metrics.items():
if t[1] < 0.1:
good_linear_metrics.append(k)
return good_linear_metrics
def find_exp_metrics(ld: LazyDictionary) -> Iterable[str]:
exp_metrics = calculate_trends(
ld["data_df"], polynomial_degree=2, nrmse_cutoff=0.05
)
good_linear_metrics = set(ld["linear_metrics"])
good_exp_metrics = []
for k, t in exp_metrics.items():
if t[1] < 0.1 and k not in good_linear_metrics:
good_exp_metrics.append(k)
return good_exp_metrics
ld = LazyDictionary()
ld["data_df"] = lambda ld: data_factory(ld)
ld["linear_metrics"] = lambda ld: find_linear_metrics(ld)
ld["exp_metrics"] = lambda ld: find_exp_metrics(ld)
# print(
# f"n_metrics == {len(data_df)} n_trending={len(linear_metrics.keys())} n_good_fit={len(good_linear_metrics)} n_good_exp={len(good_exp_metrics)}"
# )
def plot_metrics(df: pd.DataFrame, use_short_labels=False, **kwargs):
plot = (
p9.ggplot(df, p9.aes(x="date", y="value", colour="metric"))
+ p9.geom_line(size=1.3)
+ p9.geom_point(size=3)
)
if use_short_labels:
plot += p9.scale_y_continuous(labels=label_shorten)
n_metrics = df["metric"].nunique()
return user_theme(
plot,
subplots_adjust={"left": 0.2},
figure_size=(12, int(n_metrics * 1.5)),
**kwargs,
)
def plot_linear_trending_metrics(ld: LazyDictionary):
df = ld["data_df"].filter(ld["linear_metrics"], axis=0)
if len(df) < 1:
return None
df["metric"] = df.index
df = df.melt(id_vars="metric").dropna(how="any", axis=0)
plot = plot_metrics(df, use_short_labels=True)
plot += p9.facet_wrap("~metric", ncol=1, scales="free_y")
return plot
def plot_exponential_growth_metrics(ld: LazyDictionary):
df = ld["data_df"].filter(ld["exp_metrics"], axis=0)
if len(df) < 1:
return None
df["metric"] = df.index
df = df.melt(id_vars="metric").dropna(how="any", axis=0)
plot = plot_metrics(df)
plot += p9.facet_wrap("~metric", ncol=1, scales="free_y")
return plot
def plot_earnings_and_revenue(ld: LazyDictionary):
df = ld["data_df"].filter(["Ebit", "Total Revenue", "Earnings"], axis=0)
if len(df) < 2:
print(f"WARNING: revenue and earnings not availabe for {stock}")
return None
df["metric"] = df.index
df = df.melt(id_vars="metric").dropna(how="any", axis=0)
plot = plot_metrics(
df,
use_short_labels=True,
legend_position="right",
y_axis_label="$ AUD",
) # need to show metric name somewhere on plot
return plot
er_uri = cache_plot(
f"{stock}-earnings-revenue-plot",
lambda ld: plot_earnings_and_revenue(ld),
datasets=ld,
)
trending_metrics_uri = cache_plot(
f"{stock}-trending-metrics-plot",
lambda ld: plot_linear_trending_metrics(ld),
datasets=ld,
)
exp_growth_metrics_uri = cache_plot(
f"{stock}-exponential-growth-metrics-plot",
lambda ld: plot_exponential_growth_metrics(ld),
datasets=ld,
)
warning(
request,
"Due to experimental data ingest - data on this page may be wrong/misleading/inaccurate/missing. Use at own risk.",
)
context = {
"asx_code": stock,
"data": ld["data_df"],
"earnings_and_revenue_plot_uri": er_uri,
"trending_metrics_plot_uri": trending_metrics_uri,
"exp_growth_metrics_plot_uri": exp_growth_metrics_uri,
}
return render(request, "stock_financial_metrics.html", context=context)
def detect_outliers(stocks: list, all_stocks_cip: pd.DataFrame, rules=None):
"""
Returns a dataframe describing those outliers present in stocks based on the provided rules.
All_stocks_cip is the "change in percent" for at least the stocks present in the specified list
"""
if rules is None:
rules = default_point_score_rules()
str_rules = {str(r): r for r in rules}
rows = []
stocks_by_sector_df = (stocks_by_sector()
) # NB: ETFs in watchlist will have no sector
stocks_by_sector_df.index = stocks_by_sector_df["asx_code"]
for stock in stocks:
# print("Processing stock: ", stock)
try:
sector = stocks_by_sector_df.at[stock, "sector_name"]
sector_companies = list(stocks_by_sector_df.loc[
stocks_by_sector_df["sector_name"] == sector].asx_code)
# day_low_high() may raise KeyError when data is currently being fetched, so it appears here...
day_low_high_df = day_low_high(stock, all_stocks_cip.columns)
except KeyError:
warning(
None,
"Unable to locate watchlist entry: {} - continuing without it".
format(stock),
)
continue
state = {
"day_low_high_df":
day_low_high_df, # never changes each day, so we init it here
"all_stocks_change_in_percent_df": all_stocks_cip,
"stock": stock,
"daily_range_threshold":
0.20, # 20% at either end of the daily range gets a point
}
points_by_rule = defaultdict(int)
for date in all_stocks_cip.columns:
market_avg = all_stocks_cip[date].mean()
sector_avg = all_stocks_cip[date].filter(
items=sector_companies).mean()
stock_move = all_stocks_cip.at[stock, date]
state.update({
"market_avg": market_avg,
"sector_avg": sector_avg,
"stock_move": stock_move,
"date": date,
})
for rule_name, rule in str_rules.items():
try:
points_by_rule[rule_name] += rule(state)
except TypeError: # handle nan's in dataset safely
pass
d = {"stock": stock}
d.update(points_by_rule)
rows.append(d)
df = pd.DataFrame.from_records(rows)
df = df.set_index("stock")
# print(df)
clf = IForest()
clf.fit(df)
scores = clf.predict(df)
results = [
row[0] for row, value in zip(df.iterrows(), scores) if value > 0
]
# print(results)
print("Found {} outlier stocks".format(len(results)))
return results