class Job(models.Model):
class Meta:
ordering = ['created']
task_id = models.CharField(max_length=765, blank=False)
created = models.DateTimeField(auto_now_add=True)
#tables = JSONField(default=[])
_tables = models.FileField(upload_to=Locations.get_upload_path)
progress = JSONField(default={})
backend = JSONField(default={})
callback = models.CharField(max_length=255,
blank=False) # TODO: use models.URLField
@property
def tables(self):
if self._tables:
data = json.loads(self._tables.read())
self._tables.seek(0)
return data
return []
@tables.setter
def tables(self, value):
self._tables = SimpleUploadedFile("upload", json.dumps(value).encode())
class InvertedIndex(models.Model):
word = models.CharField(max_length=30, primary_key=True)
summaries = JSONField()
def __str__(self):
return 'word: {}, summaries: {}'.format(self.word,
self.summaries)
class SocialAuth(mongo.Model):
provider = mongo.CharField(max_length=32)
uid = mongo.CharField(max_length=255)
access_token = mongo.TextField(max_length=255)
extra_data = JSONField(default={})
created_on = mongo.DateTimeField()
updated_on = mongo.DateTimeField()
class Meta:
abstract = True
class WorldBankIndicators(model.Model):
id = ObjectIdField(primary_key=True, db_column="_id")
wb_id = model.TextField(
) # world bank id, not to be confused with django id/pk
name = model.TextField()
last_updated = model.DateTimeField(auto_now_add=True)
unit = model.TextField()
source = JSONField()
source_note = model.TextField()
topics = ArrayField(WorldBankTopic)
source_organisation = model.TextField()
last_successful_data = model.DateTimeField(null=True)
last_error_when = model.DateTimeField(
null=True
) # date of last ingest error for this indicator (or None if not known)
last_error_msg = model.TextField() # eg. Error code 175
last_error_type = model.TextField() # eg. class RuntimeError
objects = DjongoManager()
def __init__(self, *args, **kwargs):
self.wb_id = kwargs.pop("id", None)
self.source_organisation = kwargs.pop("sourceOrganization", None)
self.source_note = kwargs.pop("sourceNote", None)
# wb_id=wb_id, source_organisation=source_organisation, source_note=source_note
super(WorldBankIndicators, self).__init__(*args, **kwargs)
def __str__(self):
return f"{self.wb_id} {self.name} {self.source} last_error_when={self.last_error_when} last_updated={self.last_updated}"
@property
def tag(self
): # NB: must match the ingest_worldbank_datasets.py tag name...
return f"{self.wb_id}-yearly-dataframe"
@property
def has_data(self):
obj = WorldBankDataCache.objects.filter(tag=self.tag).first()
return obj is not None
def fetch_data(self) -> pd.DataFrame:
t = self.tag
print(f"Fetching parquet dataframe for {t}")
return get_parquet(t)
class Meta:
db_table = "world_bank_indicators"
verbose_name = "World Bank Metric"
verbose_name_plural = "World Bank Metrics"
class Table(models.Model):
name = models.CharField(max_length=255)
dataset = models.ForeignKey(to=Dataset,
on_delete=models.CASCADE) # TODO: check this
original = SafeJSONField(default=[])
cols = SafeJSONField(default=[])
predicates = SafeJSONField(default=[])
concepts = SafeJSONField(default=[])
linkages = JSONField(default=[]) # TODO: Just for debug and fast export
rows_count = models.PositiveIntegerField()
cols_count = models.PositiveIntegerField()
has_annotations = models.BooleanField(default=False)
class ControlHit(m.Model):
_id = ObjectIdField(db_column='_id')
control_url = m.CharField(blank=False, null=False, max_length=1024)
origin_url = m.CharField(blank=False, null=False, max_length=1024)
sha256_matched = m.BooleanField()
ast_dist = m.FloatField()
function_dist = m.FloatField()
literal_dist = m.FloatField()
cited_on = m.CharField(blank=False, null=False, max_length=1024)
#origin_js_id = m.ForeignKey(Script, on_delete=m.DO_NOTHING)
#xref = m.ForeignKey(VetAgainstControl, on_delete=m.DO_NOTHING)
origin_js_id = m.CharField(blank=False, null=False, max_length=24)
xref = m.CharField(blank=False, null=False, max_length=24)
literals_not_in_control = m.IntegerField()
literals_not_in_origin = m.IntegerField()
n_diff_literals = m.IntegerField()
diff_literals = m.CharField(null=False, max_length=1000000)
origin_host = m.CharField(max_length=255)
origin_has_query = m.BooleanField()
origin_port = m.IntegerField()
origin_scheme = m.CharField(max_length=10)
origin_path = m.CharField(max_length=1024)
cited_on_host = m.CharField(max_length=255)
cited_on_has_query = m.BooleanField()
cited_on_port = m.IntegerField()
cited_on_scheme = m.CharField(max_length=10)
cited_on_path = m.CharField(max_length=1024)
control_family = m.CharField(max_length=255)
diff_functions = JSONField()
# permit direct access to mongo for convenient API
objects = DjongoManager()
# responsibility for this data is left entirely to etl_publish_hits.py
class Meta:
db_table = "etl_hits"
managed = False
class CompanyDetails(model.Model):
# { "_id" : ObjectId("5eff01d14b1fe020d5453e8f"), "asx_code" : "NIC", "delisting_date" : null,
# "fax_number" : "02 9221 6333", "fiscal_year_end" : "31/12", "foreign_exempt" : false,
# "industry_group_name" : "Materials",
# "latest_annual_reports" : [ { "id" : "02229616", "document_release_date" : "2020-04-29T14:45:12+1000",
# "document_date" : "2020-04-29T14:39:36+1000", "url" : "http://www.asx.com.au/asxpdf/20200429/pdf/44hc5731pmh9mw.pdf", "relative_url" : "/asxpdf/20200429/pdf/44hc5731pmh9mw.pdf", "header" : "Annual Report and Notice of AGM", "market_sensitive" : false, "number_of_pages" : 118, "size" : "4.0MB", "legacy_announcement" : false }, { "id" : "02209126", "document_release_date" : "2020-02-28T18:09:26+1100", "document_date" : "2020-02-28T18:06:25+1100", "url" : "http://www.asx.com.au/asxpdf/20200228/pdf/44fm8tp5qy0k7x.pdf", "relative_url" : "/asxpdf/20200228/pdf/44fm8tp5qy0k7x.pdf",
# "header" : "Annual Report and Appendix 4E", "market_sensitive" : true, "number_of_pages" : 64,
# "size" : "1.6MB", "legacy_announcement" : false }, { "id" : "02163933", "document_release_date" :
# "2019-10-25T11:50:50+1100", "document_date" : "2019-10-25T11:48:43+1100",
# "url" : "http://www.asx.com.au/asxpdf/20191025/pdf/449w6d0phvgr05.pdf",
# "relative_url" : "/asxpdf/20191025/pdf/449w6d0phvgr05.pdf", "header" : "Annual Report and Notice of AGM",
# "market_sensitive" : false, "number_of_pages" : 74, "size" : "2.5MB", "legacy_announcement" : false } ],
# "listing_date" : "2018-08-20T00:00:00+1000",
# "mailing_address" : "Level 2, 66 Hunter Street, SYDNEY, NSW, AUSTRALIA, 2000",
# "name_abbrev" : "NICKELMINESLIMITED", "name_full" : "NICKEL MINES LIMITED",
# "name_short" : "NICKLMINES", "phone_number" : "02 9300 3311",
# "primary_share" : { "code" : "NIC", "isin_code" : "AU0000018236", "desc_full" : "Ordinary Fully Paid",
# "last_price" : 0.61, "open_price" : 0.595, "day_high_price" : 0.615, "day_low_price" : 0.585,
# "change_price" : 0.02, "change_in_percent" : "3.39%", "volume" : 3127893, "bid_price" : 0.605,
# "offer_price" : 0.61, "previous_close_price" : 0.59, "previous_day_percentage_change" : "1.724%",
# "year_high_price" : 0.731, "last_trade_date" : "2020-07-24T00:00:00+1000",
# "year_high_date" : "2019-09-24T00:00:00+1000", "year_low_price" : 0.293,
# "year_low_date" : "2020-03-18T00:00:00+1100", "pe" : 8.73, "eps" : 0.0699,
# "average_daily_volume" : 7504062, "annual_dividend_yield" : 0, "market_cap" : 1255578789,
# "number_of_shares" : 2128099642, "deprecated_market_cap" : 1127131000,
# "deprecated_number_of_shares" : 1847755410, "suspended" : false,
# "indices" : [ { "index_code" : "XKO", "name_full" : "S&P/ASX 300", "name_short" : "S&P/ASX300",
# "name_abrev" : "S&P/ASX 300" }, { "index_code" : "XAO", "name_full" : "ALL ORDINARIES",
# "name_short" : "ALL ORDS", "name_abrev" : "All Ordinaries" }, { "index_code" : "XSO",
# "name_full" : "S&P/ASX SMALL ORDINARIES", "name_short" : "Small Ords",
# "name_abrev" : "S&P/ASX Small Ords" }, { "index_code" : "XMM",
# "name_full" : "S&P/ASX 300 Metals and Mining (Industry)", "name_short" : "MTL&MINING",
# "name_abrev" : "Metals and Mining" } ] }, "primary_share_code" : "NIC",
# "principal_activities" : "Nickel pig iron and nickel ore production.",
# "products" : [ "shares" ], "recent_announcement" : false,
# "registry_address" : "Level 3, 60 Carrington Street, SYDNEY, NSW, AUSTRALIA, 2000",
# "registry_name" : "COMPUTERSHARE INVESTOR SERVICES PTY LIMITED",
# "registry_phone_number" : "02 8234 5000", "sector_name" : "Metals & Mining",
# "web_address" : "http://www.nickelmines.com.au/" }
_id = ObjectIdField()
asx_code = model.TextField(null=False, blank=False)
delisting_date = model.TextField(null=True) # if null, not delisted
name_full = model.TextField(null=False, blank=False)
phone_number = model.TextField(null=False, blank=True)
bid_price = model.FloatField()
offer_price = model.FloatField()
latest_annual_reports = JSONField()
previous_close_price = model.FloatField()
average_daily_volume = model.IntegerField()
number_of_shares = model.IntegerField()
suspended = model.BooleanField()
indices = JSONField()
primary_share_code = model.TextField()
principal_activities = model.TextField()
products = JSONField()
recent_announcement = model.BooleanField()
registry_address = model.TextField()
registry_name = model.TextField()
registry_phone_number = model.TextField()
sector_name = model.TextField()
web_address = model.TextField()
objects = DjongoManager()
class Meta:
managed = False
db_table = "asx_company_details"
verbose_name = "Company Detail"
verbose_name_plural = "Company Details"