class MarketQuoteCache(model.Model):
# { "_id" : ObjectId("5f44c54457d4bb6dfe6b998f"), "scope" : "all-downloaded",
# "tag" : "change_price-05-2020-asx", "dataframe_format" : "parquet",
# "field" : "change_price", "last_updated" : ISODate("2020-08-25T08:01:08.804Z"),
# "market" : "asx", "n_days" : 3, "n_stocks" : 0,
# "sha256" : "75d0ad7e057621e6a73508a178615bcc436d97110bcc484f1cfb7d478475abc5",
# "size_in_bytes" : 2939, "status" : "INCOMPLETE" }
size_in_bytes = model.IntegerField()
status = model.TextField()
tag = model.TextField(db_index=True)
dataframe_format = model.TextField()
field = model.TextField()
last_updated = model.DateTimeField()
market = model.TextField()
n_days = model.IntegerField()
n_stocks = model.IntegerField()
sha256 = model.TextField()
_id = ObjectIdField()
scope = model.TextField()
dataframe = model.BinaryField()
objects = DjongoManager()
class Meta:
managed = False # table is managed by persist_dataframes.py
db_table = "market_quote_cache"
class ECBMetadata(models.Model):
"""
Model responsible for describing allowable values for a given metadata_type (dimension, attribute, measure)
which is used for form generation and validation
"""
id = ObjectIdField(db_column="_id", primary_key=True)
metadata_type = models.TextField(db_column="metadata_type",
null=False,
blank=False)
flow = models.TextField(db_column="flow_name", null=False, blank=False)
code = models.TextField(db_column="item_name", null=False, blank=False)
column_name = models.TextField(db_column="codelist_name",
null=False,
blank=False)
printable_code = models.TextField(db_column="item_value",
null=False,
blank=False)
objects = DjongoManager()
class Meta:
db_table = "ecb_metadata_index"
verbose_name = "ECB Metadata"
verbose_name_plural = "ECB Metadata"
class VirtualPurchase(model.Model):
id = ObjectIdField(unique=True, db_column="_id")
user = model.ForeignKey(settings.AUTH_USER_MODEL, on_delete=model.CASCADE)
asx_code = model.TextField(max_length=10)
buy_date = model.DateField()
price_at_buy_date = model.FloatField()
amount = model.FloatField() # dollar value purchased (assumes no fees)
n = (
model.IntegerField()
) # number of shares purchased at buy_date (rounded down to nearest whole share)
objects = DjongoManager()
def current_price(self):
assert self.n > 0
quote, _ = latest_quote(self.asx_code)
if quote is None:
raise ValueError()
buy_price = self.price_at_buy_date
pct_move = (quote.last_price / buy_price) * 100.0 - 100.0 if buy_price > 0.0 else 0.0
return (self.n * quote.last_price, pct_move)
def __str__(self):
cur_price, pct_move = self.current_price()
return "Purchase on {}: ${} ({} shares@${:.2f}) is now ${:.2f} ({:.2f}%)".format(
self.buy_date, self.amount, self.n, self.price_at_buy_date, cur_price, pct_move
)
class Meta:
managed = True # viewer application
db_table = "virtual_purchase"
class Post(Model):
title = CharField(max_length=255)
slug = SlugField(unique=True)
content = TextField()
is_draft = BooleanField(default=True)
is_archived = BooleanField(default=False)
published = DateTimeField(null=True)
authors = ManyToManyField(User)
tags = ArrayModelField(model_container=Tag)
created = DateTimeField(auto_now_add=True)
updated = DateTimeField(auto_now=True)
# cover_img
# authors (multiple)
# categories
# tags
# layout
# allow_comments
objects = DjongoManager()
def __str__(self):
return self.title
class JSONAPIMeta:
resource_name = "posts"
class CompanyFinancialMetric(model.Model):
id = ObjectIdField(db_column="_id", primary_key=True)
asx_code = model.TextField(blank=False, null=False)
name = model.TextField(blank=False, null=False, db_column="metric")
value = model.FloatField()
as_at = model.DateTimeField(db_column="date")
objects = DjongoManager()
class Meta:
db_table = "asx_company_financial_metrics"
class ABSHeadlineDataRecord(model.Model):
dataflow = model.TextField()
idx = model.IntegerField(blank=False, null=False)
time_period = model.TextField(null=False, blank=False)
variable = model.TextField(null=False, blank=False)
val = model.TextField()
objects = DjongoManager()
class Meta:
db_table = "abs_headline_data"
class Watchlist(model.Model):
id = ObjectIdField(unique=True, db_column="_id")
# record stocks of interest to the user
user = model.ForeignKey(settings.AUTH_USER_MODEL, on_delete=model.CASCADE)
asx_code = model.TextField()
when = model.DateTimeField(auto_now_add=True)
objects = DjongoManager()
class Meta:
managed = True # viewer application is responsible NOT asxtrade.py
db_table = "user_watchlist"
class SEOResult(models.Model):
query = models.TextField()
text = models.TextField()
score = models.FloatField()
query_keywords = models.ListField(default=[])
document_keywords = models.ListField(default=[])
general = models.ListField(default=[])
specific = models.ListField(default=[])
objects = DjongoManager()
class Meta:
abstract = True
class Sector(model.Model):
"""
Table of ASX sector (GICS) names. Manually curated for now.
"""
id = ObjectIdField(unique=True, db_column="_id")
sector_name = model.TextField(unique=True)
sector_id = model.IntegerField(db_column="id")
objects = DjongoManager()
class Meta:
managed = False
db_table = "sector"
class WorldBankIndicators(model.Model):
id = ObjectIdField(primary_key=True, db_column="_id")
wb_id = model.TextField(
) # world bank id, not to be confused with django id/pk
name = model.TextField()
last_updated = model.DateTimeField(auto_now_add=True)
unit = model.TextField()
source = JSONField()
source_note = model.TextField()
topics = ArrayField(WorldBankTopic)
source_organisation = model.TextField()
last_successful_data = model.DateTimeField(null=True)
last_error_when = model.DateTimeField(
null=True
) # date of last ingest error for this indicator (or None if not known)
last_error_msg = model.TextField() # eg. Error code 175
last_error_type = model.TextField() # eg. class RuntimeError
objects = DjongoManager()
def __init__(self, *args, **kwargs):
self.wb_id = kwargs.pop("id", None)
self.source_organisation = kwargs.pop("sourceOrganization", None)
self.source_note = kwargs.pop("sourceNote", None)
# wb_id=wb_id, source_organisation=source_organisation, source_note=source_note
super(WorldBankIndicators, self).__init__(*args, **kwargs)
def __str__(self):
return f"{self.wb_id} {self.name} {self.source} last_error_when={self.last_error_when} last_updated={self.last_updated}"
@property
def tag(self
): # NB: must match the ingest_worldbank_datasets.py tag name...
return f"{self.wb_id}-yearly-dataframe"
@property
def has_data(self):
obj = WorldBankDataCache.objects.filter(tag=self.tag).first()
return obj is not None
def fetch_data(self) -> pd.DataFrame:
t = self.tag
print(f"Fetching parquet dataframe for {t}")
return get_parquet(t)
class Meta:
db_table = "world_bank_indicators"
verbose_name = "World Bank Metric"
verbose_name_plural = "World Bank Metrics"
class ABSInvertedIndex(model.Model):
id = ObjectIdField(primary_key=True, db_column="_id")
min_date = model.DateTimeField()
max_date = model.DateTimeField()
n_attributes = model.IntegerField()
last_updated = model.DateTimeField(auto_now_add=True)
tag = model.TextField()
dataset = model.TextField()
name = model.TextField()
scope = model.TextField() # always 'abs' for now
objects = DjongoManager()
class Meta:
db_table = "abs_inverted_index"
class WorldBankCountry(model.Model):
"""
Not strictly a country, can include regions eg. africa ex-warzones
"""
id = ObjectIdField(primary_key=True, db_column="_id")
country_code = model.TextField()
name = model.TextField()
last_updated = model.DateTimeField(auto_now_add=True)
objects = DjongoManager()
class Meta:
db_table = "world_bank_countries"
verbose_name_plural = "World Bank Countries"
class WorldBankInvertedIndex(model.Model):
id = ObjectIdField(primary_key=True, db_column="_id")
country = model.TextField()
topic_id = model.IntegerField()
topic_name = model.TextField()
n_attributes = model.IntegerField()
last_updated = model.DateTimeField()
tag = model.TextField()
indicator_id = model.TextField() # xref into WorldBankIndicators.wb_id
indicator_name = model.TextField() # likewise to name field
objects = DjongoManager()
class Meta:
db_table = "world_bank_inverted_index"
verbose_name_plural = "World Bank Inverted Indexes"
class Security(model.Model):
# eg. { "_id" : ObjectId("5efe83dd4b1fe020d5ba2de8"), "asx_code" : "T3DAC",
# "asx_isin_code" : "AU0000T3DAC0", "company_name" : "333D LIMITED",
# "last_updated" : ISODate("2020-07-26T00:49:11.052Z"),
# "security_name" : "OPTION EXPIRING 18-AUG-2018 RESTRICTED" }
_id = ObjectIdField()
asx_code = model.TextField(blank=False, null=False)
asx_isin_code = model.TextField()
company_name = model.TextField()
last_updated = model.DateField()
security_name = model.TextField()
objects = DjongoManager()
class Meta:
db_table = "asx_isin"
managed = False # managed by asxtrade.py
verbose_name = "Security"
verbose_name_plural = "Securities"
class WorldBankTopic(model.Model):
id = model.IntegerField(null=False, blank=False, primary_key=True)
topic = model.TextField(null=False, blank=False)
source_note = model.TextField(null=False, blank=False)
last_updated = model.DateTimeField(auto_now_add=True)
objects = DjongoManager()
def __init__(self, *args,
**kwargs): # support initialization via ArrayField
topic = kwargs.pop("value", None)
source_note = kwargs.pop("sourceNote", None)
extra_args = {}
if topic is not None:
extra_args["topic"] = topic
if source_note is not None:
extra_args["source_note"] = source_note
super(WorldBankTopic, self).__init__(*args, **kwargs, **extra_args)
class Meta:
db_table = "world_bank_topics"
class ECBDataCache(models.Model):
""" Similar to worldbank data cache """
id = ObjectIdField(db_column="_id", primary_key=True)
size_in_bytes = models.IntegerField()
status = models.TextField()
tag = models.TextField()
dataframe_format = models.TextField()
field = models.TextField()
last_updated = models.DateTimeField()
market = models.TextField()
n_days = models.IntegerField()
n_stocks = models.IntegerField()
sha256 = models.TextField()
scope = models.TextField()
dataframe = models.BinaryField()
objects = DjongoManager()
class Meta:
db_table = "ecb_data_cache"
verbose_name = "ECB Data Cache"
verbose_name_plural = "ECB Data Cache"
class ECBFlow(models.Model):
"""
Describe each dataflow ingested from ECB SDMX REST API
"""
id = ObjectIdField(db_column="_id", primary_key=True)
name = models.TextField(db_column="flow_name", null=False, blank=False)
description = models.TextField(db_column="flow_descr",
null=False,
blank=False)
is_test_data = models.BooleanField()
last_updated = models.DateTimeField()
prepared = models.DateTimeField()
sender = models.TextField()
source = models.TextField()
data_available = models.BooleanField()
objects = DjongoManager()
class Meta:
db_table = "ecb_flow_index"
verbose_name = "ECB Flow"
verbose_name_plural = "ECB Flow"
class ControlHit(m.Model):
_id = ObjectIdField(db_column='_id')
control_url = m.CharField(blank=False, null=False, max_length=1024)
origin_url = m.CharField(blank=False, null=False, max_length=1024)
sha256_matched = m.BooleanField()
ast_dist = m.FloatField()
function_dist = m.FloatField()
literal_dist = m.FloatField()
cited_on = m.CharField(blank=False, null=False, max_length=1024)
#origin_js_id = m.ForeignKey(Script, on_delete=m.DO_NOTHING)
#xref = m.ForeignKey(VetAgainstControl, on_delete=m.DO_NOTHING)
origin_js_id = m.CharField(blank=False, null=False, max_length=24)
xref = m.CharField(blank=False, null=False, max_length=24)
literals_not_in_control = m.IntegerField()
literals_not_in_origin = m.IntegerField()
n_diff_literals = m.IntegerField()
diff_literals = m.CharField(null=False, max_length=1000000)
origin_host = m.CharField(max_length=255)
origin_has_query = m.BooleanField()
origin_port = m.IntegerField()
origin_scheme = m.CharField(max_length=10)
origin_path = m.CharField(max_length=1024)
cited_on_host = m.CharField(max_length=255)
cited_on_has_query = m.BooleanField()
cited_on_port = m.IntegerField()
cited_on_scheme = m.CharField(max_length=10)
cited_on_path = m.CharField(max_length=1024)
control_family = m.CharField(max_length=255)
diff_functions = JSONField()
# permit direct access to mongo for convenient API
objects = DjongoManager()
# responsibility for this data is left entirely to etl_publish_hits.py
class Meta:
db_table = "etl_hits"
managed = False
class ABSDataCache(model.Model):
"""
Similar to, but separate from, app.MarketQuoteCache, this keeps track of pandas dataframes (parquet format) which have
been downloaded, cleaned and ingested into MongoDB
"""
size_in_bytes = model.IntegerField()
status = model.TextField()
tag = model.TextField()
dataframe_format = model.TextField()
field = model.TextField()
last_updated = model.DateTimeField()
market = model.TextField()
n_days = model.IntegerField()
n_stocks = model.IntegerField()
sha256 = model.TextField()
_id = ObjectIdField()
scope = model.TextField()
dataframe = model.BinaryField()
objects = DjongoManager()
class Meta:
db_table = "abs_data_cache"
class ABSDataflow(model.Model):
abs_id = model.TextField()
name = model.TextField()
objects = DjongoManager()
class CompanyDetails(model.Model):
# { "_id" : ObjectId("5eff01d14b1fe020d5453e8f"), "asx_code" : "NIC", "delisting_date" : null,
# "fax_number" : "02 9221 6333", "fiscal_year_end" : "31/12", "foreign_exempt" : false,
# "industry_group_name" : "Materials",
# "latest_annual_reports" : [ { "id" : "02229616", "document_release_date" : "2020-04-29T14:45:12+1000",
# "document_date" : "2020-04-29T14:39:36+1000", "url" : "http://www.asx.com.au/asxpdf/20200429/pdf/44hc5731pmh9mw.pdf", "relative_url" : "/asxpdf/20200429/pdf/44hc5731pmh9mw.pdf", "header" : "Annual Report and Notice of AGM", "market_sensitive" : false, "number_of_pages" : 118, "size" : "4.0MB", "legacy_announcement" : false }, { "id" : "02209126", "document_release_date" : "2020-02-28T18:09:26+1100", "document_date" : "2020-02-28T18:06:25+1100", "url" : "http://www.asx.com.au/asxpdf/20200228/pdf/44fm8tp5qy0k7x.pdf", "relative_url" : "/asxpdf/20200228/pdf/44fm8tp5qy0k7x.pdf",
# "header" : "Annual Report and Appendix 4E", "market_sensitive" : true, "number_of_pages" : 64,
# "size" : "1.6MB", "legacy_announcement" : false }, { "id" : "02163933", "document_release_date" :
# "2019-10-25T11:50:50+1100", "document_date" : "2019-10-25T11:48:43+1100",
# "url" : "http://www.asx.com.au/asxpdf/20191025/pdf/449w6d0phvgr05.pdf",
# "relative_url" : "/asxpdf/20191025/pdf/449w6d0phvgr05.pdf", "header" : "Annual Report and Notice of AGM",
# "market_sensitive" : false, "number_of_pages" : 74, "size" : "2.5MB", "legacy_announcement" : false } ],
# "listing_date" : "2018-08-20T00:00:00+1000",
# "mailing_address" : "Level 2, 66 Hunter Street, SYDNEY, NSW, AUSTRALIA, 2000",
# "name_abbrev" : "NICKELMINESLIMITED", "name_full" : "NICKEL MINES LIMITED",
# "name_short" : "NICKLMINES", "phone_number" : "02 9300 3311",
# "primary_share" : { "code" : "NIC", "isin_code" : "AU0000018236", "desc_full" : "Ordinary Fully Paid",
# "last_price" : 0.61, "open_price" : 0.595, "day_high_price" : 0.615, "day_low_price" : 0.585,
# "change_price" : 0.02, "change_in_percent" : "3.39%", "volume" : 3127893, "bid_price" : 0.605,
# "offer_price" : 0.61, "previous_close_price" : 0.59, "previous_day_percentage_change" : "1.724%",
# "year_high_price" : 0.731, "last_trade_date" : "2020-07-24T00:00:00+1000",
# "year_high_date" : "2019-09-24T00:00:00+1000", "year_low_price" : 0.293,
# "year_low_date" : "2020-03-18T00:00:00+1100", "pe" : 8.73, "eps" : 0.0699,
# "average_daily_volume" : 7504062, "annual_dividend_yield" : 0, "market_cap" : 1255578789,
# "number_of_shares" : 2128099642, "deprecated_market_cap" : 1127131000,
# "deprecated_number_of_shares" : 1847755410, "suspended" : false,
# "indices" : [ { "index_code" : "XKO", "name_full" : "S&P/ASX 300", "name_short" : "S&P/ASX300",
# "name_abrev" : "S&P/ASX 300" }, { "index_code" : "XAO", "name_full" : "ALL ORDINARIES",
# "name_short" : "ALL ORDS", "name_abrev" : "All Ordinaries" }, { "index_code" : "XSO",
# "name_full" : "S&P/ASX SMALL ORDINARIES", "name_short" : "Small Ords",
# "name_abrev" : "S&P/ASX Small Ords" }, { "index_code" : "XMM",
# "name_full" : "S&P/ASX 300 Metals and Mining (Industry)", "name_short" : "MTL&MINING",
# "name_abrev" : "Metals and Mining" } ] }, "primary_share_code" : "NIC",
# "principal_activities" : "Nickel pig iron and nickel ore production.",
# "products" : [ "shares" ], "recent_announcement" : false,
# "registry_address" : "Level 3, 60 Carrington Street, SYDNEY, NSW, AUSTRALIA, 2000",
# "registry_name" : "COMPUTERSHARE INVESTOR SERVICES PTY LIMITED",
# "registry_phone_number" : "02 8234 5000", "sector_name" : "Metals & Mining",
# "web_address" : "http://www.nickelmines.com.au/" }
_id = ObjectIdField()
asx_code = model.TextField(null=False, blank=False)
delisting_date = model.TextField(null=True) # if null, not delisted
name_full = model.TextField(null=False, blank=False)
phone_number = model.TextField(null=False, blank=True)
bid_price = model.FloatField()
offer_price = model.FloatField()
latest_annual_reports = JSONField()
previous_close_price = model.FloatField()
average_daily_volume = model.IntegerField()
number_of_shares = model.IntegerField()
suspended = model.BooleanField()
indices = JSONField()
primary_share_code = model.TextField()
principal_activities = model.TextField()
products = JSONField()
recent_announcement = model.BooleanField()
registry_address = model.TextField()
registry_name = model.TextField()
registry_phone_number = model.TextField()
sector_name = model.TextField()
web_address = model.TextField()
objects = DjongoManager()
class Meta:
managed = False
db_table = "asx_company_details"
verbose_name = "Company Detail"
verbose_name_plural = "Company Details"
class Quotation(model.Model):
_id = ObjectIdField()
error_code = model.TextField(
max_length=100) # ignore record iff set to non-empty
error_descr = model.TextField(max_length=100)
fetch_date = model.TextField(
null=False, blank=False) # used as an index (string) always YYYY-mm-dd
asx_code = model.TextField(blank=False, null=False, max_length=20)
annual_dividend_yield = model.FloatField()
average_daily_volume = model.IntegerField()
bid_price = model.FloatField()
change_in_percent = model.FloatField()
change_price = model.FloatField()
code = model.TextField(blank=False, null=False, max_length=20)
day_high_price = model.FloatField()
day_low_price = model.FloatField()
deprecated_market_cap = (model.IntegerField()
) # NB: deprecated use market_cap instead
deprecated_number_of_shares = model.IntegerField()
descr_full = model.TextField(max_length=100) # eg. Ordinary Fully Paid
eps = model.FloatField()
isin_code = model.TextField(max_length=100)
last_price = model.FloatField()
last_trade_date = model.DateField()
market_cap = model.IntegerField()
number_of_shares = model.IntegerField()
offer_price = model.FloatField()
open_price = model.FloatField()
pe = model.FloatField()
previous_close_price = model.FloatField()
previous_day_percentage_change = model.FloatField()
suspended = model.BooleanField(default=False)
volume = model.IntegerField()
year_high_date = model.DateField()
year_high_price = model.FloatField()
year_low_date = model.DateField()
year_low_price = model.FloatField()
objects = DjongoManager() # convenient access to mongo API
def __str__(self):
assert self is not None
return str(model_to_dict(self))
def is_error(self):
if self.error_code is None:
return False
return len(self.error_code) > 0
def eps_as_cents(self):
if any([self.is_error(), self.eps is None]):
return 0.0
return self.eps * 100.0
def volume_as_millions(self):
"""
Return the volume as a formatted string (rounded to 2 decimal place)
represent the millions of dollars transacted for a given quote
"""
if any([self.is_error(), self.volume is None,
self.last_price is None]):
return ""
return "{:.2f}".format(self.volume * self.last_price / 1000000.0)
class Meta:
db_table = "asx_prices"
managed = False # managed by asxtrade.py
indexes = [
model.Index(fields=["asx_code"]),
model.Index(fields=["fetch_date"])
]