def __init__(self):
self.target = NamedStatisticsByRegionAndDate(
service_url=r'https://services9.arcgis.com/7eQuDPPaB6g9029K/arcgis/rest/services/COVID_19_Statistics_by_Postcode_and_Date/FeatureServer/0',
region_id_field='Postcode',
statistic_name='TESTS'
)
self.nsw_source = NswNotificationData()
self.geometry_source = POA2016.centroids()
def update_from_source(self,
nsw_source: NSW_SourceData.NswNotificationData = NSW_SourceData.NswNotificationData(),
geometry_source: TableBase = None):
new_records = {}
# get state capital geometries indexed by state abbreviation
if not geometry_source:
geometry_source = POA2016.centroids()
features = geometry_source.values_by_id(value_field='SHAPE@')
# load nsw source. NSW source values override any existing jhu values for nsw.
for key, case_count in nsw_source.counts_by_date_and_postcode().items():
date_code, postcode = key.split('_')
new_record = new_records.get(key, None)
if new_record:
new_record[self.cases_field] = case_count
else:
shape_value = features.get(postcode, None)
date_value = datetime.datetime.strptime(date_code, '%Y%m%d').date()
new_records[key] = {self.id_field: key,
self.postcode_field: postcode,
self.date_field: date_value,
self.cases_field: case_count,
self.shape_field: shape_value}
fields = [self.id_field, self.date_field, self.postcode_field, self.cases_field]
return self._helper.update_records(new_data=new_records, fields=fields, add_new=True, delete_unmatched=True, shape_field=self.shape_field)
def update_from_source(self,
covid_source: covid19data_SourceData.COVID_AU_state = covid19data_SourceData.COVID_AU_state(),
nsw_source: NSW_SourceData.NswNotificationData = NSW_SourceData.NswNotificationData(),
geometry_source: TableBase = None):
new_records = {}
# get state capital geometries indexed by state abbreviation
if not geometry_source:
geometry_source = StateCapitals()
features = geometry_source.values_by_id(value_field='SHAPE@')
for item in covid_source.source_data():
date_value = item[covid_source.fields.date]
state = item[covid_source.fields.state_abbrev]
case_count = item[covid_source.fields.confirmed]
date_code = date_value.strftime('%Y%m%d')
shape_value = features.get(state, None)
key = '{}_{}'.format(date_code, state)
new_records[key] = {self.id_field: key,
self.state_field: state,
self.date_field: date_value,
self.cases_field: case_count,
self.shape_field: shape_value}
# load nsw source. NSW source values override any existing jhu values for nsw.
state = 'NSW'
for date_code, case_count in nsw_source.counts_by_date().items():
key = '{}_{}'.format(date_code, state)
new_record = new_records.get(key, None)
if new_record:
new_record[self.cases_field] = case_count
else:
date_value = datetime.datetime.strptime(date_code, '%Y%m%d')
shape_value = features.get(state, None)
new_records[key] = {self.id_field: key,
self.state_field: state,
self.date_field: date_value,
self.cases_field: case_count,
self.shape_field: shape_value}
aus_records = {}
for date_state, record in new_records.items():
date_code, state = date_state.split('_')
key = '{}_AUS'.format(date_code)
aus_record = aus_records.get(key, None)
if aus_record:
aus_record[self.cases_field] += record[self.cases_field]
else:
date_value = datetime.datetime.strptime(date_code, '%Y%m%d')
aus_records[key] = {self.id_field: key,
self.state_field: 'AUS',
self.date_field: date_value,
self.cases_field: record[self.cases_field],
self.shape_field: None}
for key, record in aus_records.items():
new_records[key] = record
fields = [self.id_field, self.date_field, self.state_field, self.cases_field]
return self._helper.update_records(new_data=new_records, fields=fields, add_new=True, delete_unmatched=True, shape_field=self.shape_field)
def update_from_source(self,
jhu_source: JHU_SourceData.JhuCasesData = JHU_SourceData.JhuCasesData(),
nsw_source: NSW_SourceData.NswNotificationData = NSW_SourceData.NswNotificationData(),
geometry_source: TableBase = None):
new_records = {}
# get state capital geometries indexed by state abbreviation
if not geometry_source:
geometry_source = StateCapitals()
features = geometry_source.values_by_id(value_field='SHAPE@')
# load jhu source first. Most states do not have their own source.
jhu_data = jhu_source.counts_by_date_and_state(use_abbreviation=True)
for key, case_count in jhu_data.items():
date_code, state = key.split('_')
date_value = datetime.datetime.strptime(date_code, '%Y%m%d')
shape_value = features.get(state, None)
new_records[key] = {self.id_field: key,
self.state_field: state,
self.date_field: date_value,
self.cases_field: case_count,
self.shape_field: shape_value}
# load nsw source. NSW source values override any existing jhu values for nsw.
state = 'NSW'
for date_code, case_count in nsw_source.counts_by_date().items():
key = '{}_{}'.format(date_code, state)
new_record = new_records.get(key, None)
if new_record:
new_record[self.cases_field] = case_count
else:
date_value = datetime.datetime.strptime(date_code, '%Y%m%d')
shape_value = features.get(state, None)
new_records[key] = {self.id_field: key,
self.state_field: state,
self.date_field: date_value,
self.cases_field: case_count,
self.shape_field: shape_value}
aus_records = {}
for date_state, record in new_records.items():
date_code, state = date_state.split('_')
key = '{}_AUS'.format(date_code)
aus_record = aus_records.get(key, None)
if aus_record:
aus_record[self.cases_field] += record[self.cases_field]
else:
date_value = datetime.datetime.strptime(date_code, '%Y%m%d')
aus_records[key] = {self.id_field: key,
self.state_field: 'AUS',
self.date_field: date_value,
self.cases_field: record[self.cases_field],
self.shape_field: None}
for key, record in aus_records.items():
new_records[key] = record
fields = [self.id_field, self.date_field, self.state_field, self.cases_field]
return self._helper.update_records(new_data=new_records, fields=fields, add_new=True, delete_unmatched=True, shape_field=self.shape_field)
class UpdateFromSourceTask(object):
"""
An update task encapsulating all of the required input parameters and structures.
Upon initial creation the defaults are set.
Defaults can be modified after creation using the class properties.
"""
def __init__(self):
self.feature_service = Covid19NotificationsByDateAndPostcode()
self.nsw_notifications_source = NswNotificationData()
self.postcode_feature_source = POA2016.centroids()
def full_update(self):
"""
Performs a full update from the source data, including adds, deletes and updates.
:return:
:rtype:
"""
# get the source data
source_data = self.nsw_notifications_source.source_data()
# sort rows by date and postcode.
sorted_data = sorted(source_data,
key=lambda x:
(x[self.nsw_notifications_source.postcode_field],
x[self.nsw_notifications_source.date_field]))
# get postcode xys
xys = da_arcpy.load_xy_geometries(
source=self.postcode_feature_source.source,
id_field=self.postcode_feature_source.poa_code_field)
# convert to input rows
wkg = {}
for row in sorted_data:
date_str = self.feature_service.format_date_string(
date_string=row[self.nsw_notifications_source.date_field],
in_format='%Y%m%d')
postcode = row[self.nsw_notifications_source.postcode_field]
if not postcode:
postcode = 'UNK'
key = '{}_{}'.format(date_str, postcode)
wkg_item = wkg.get(key, None)
if wkg_item:
wkg_item['notifications'] += 1
else:
xy = xys.get(postcode, None)
wkg[key] = {'notifications': 1, 'xy': xy}
return self.feature_service.update(update_values=wkg,
allow_deletes=True)
class Covid19TestsByPostcodeAndDate(object):
def __init__(self):
self.target = NamedStatisticsByRegionAndDate(
service_url=r'https://services9.arcgis.com/7eQuDPPaB6g9029K/arcgis/rest/services/COVID_19_Statistics_by_Postcode_and_Date/FeatureServer/0',
region_id_field='Postcode',
statistic_name='TESTS'
)
self.nsw_source = NswNotificationData()
self.geometry_source = POA2016.centroids()
def update(self):
# get the source data
source_data = self.nsw_source.source_data()
# get postcode xys
xys = da_arcpy.load_xy_geometries(source=self.geometry_source._source,
id_field=self.geometry_source.poa_code_field)
# convert to input rows
data = {}
for row in source_data:
date_str = self.target.format_date_string(date_string=row[self.nsw_source.date_field], in_format=self.nsw_source.csv_date_format)
postcode = row[self.nsw_source.postcode_field]
if not postcode:
postcode = 'UNK'
key = '{}_{}'.format(date_str, postcode)
data_item = data.get(key, None)
if data_item:
data_item['value'] += 1
else:
xy = xys.get(postcode, None)
data[key] = {'value': 1, 'xy': xy}
return self.target.update_statistic(update_values=data, allow_deletes=True)
def __init__(self):
self.feature_service = Covid19NotificationsByDateAndPostcode()
self.nsw_notifications_source = NswNotificationData()
self.postcode_feature_source = POA2016.centroids()
def __init__(self, service_url=default_service_url):
self.target = DailyCovid19CasesByLGA(service_url)
self.nsw_source = NswNotificationData()
self.vic_source = VicData()
self.lga2019 = LGA2019.centroids()
self.lga2020 = LGA2020.centroids()
class UpdateFromSourceTask(object):
"""
An update task encapsulating all of the required input parameters and structures.
Upon initial creation the defaults are set.
Defaults can be modified after creation using through the class properties.
"""
def __init__(self, service_url=default_service_url):
self.target = DailyCovid19CasesByLGA(service_url)
self.nsw_source = NswNotificationData()
self.vic_source = VicData()
self.lga2019 = LGA2019.centroids()
self.lga2020 = LGA2020.centroids()
def _get_nsw_source(self):
# get the source data
source_data = self.nsw_source.source_data()
# sort rows by date and lga_code.
sorted_data = sorted(
source_data,
key=lambda x:
(x[self.nsw_source.lga_code_field], x[self.nsw_source.date_field]))
# convert to input rows
xys = da_arcpy.load_xy_geometries(source=self.lga2019._source,
id_field=self.lga2019.lga_code_field,
where_clause="{} = '1'".format(
self.lga2019.ste_code_field))
totals = {}
wkg = {}
for row in sorted_data:
date_str = self.target.format_date_string(
date_string=row[self.nsw_source.date_field],
in_format=self.nsw_source.csv_date_format)
lga_code = row[self.nsw_source.lga_code_field]
if not lga_code:
lga_code = '00000'
total = totals.get(lga_code, 0) + 1
totals[lga_code] = total
key = '{}_{}'.format(lga_code, date_str)
wkg_item = wkg.get(key, None)
if wkg_item:
wkg_item['cases'] += 1
wkg_item['total_cases'] = total
else:
lga_name = row[self.nsw_source.lga_name_field]
wkg_item = {
'lga_code': lga_code,
'lga_name': lga_name,
'date_code': date_str,
'cases': 1,
'total_cases': total,
'lga_version': '2019',
'xy': xys.get(lga_code, None)
}
wkg[key] = wkg_item
return wkg.values()
def update_nsw(self):
source = self._get_nsw_source()
return self.target.update(rows=source, allow_deletes=False)
def _get_vic_source(self):
source_data = self.vic_source.daily_cases_by_lga()
xys = da_arcpy.load_xy_geometries(source=self.lga2020._source,
id_field=self.lga2020.lga_code_field,
where_clause="{} = '2'".format(
self.lga2020.ste_code_field))
for source_item in source_data:
source_id = source_item['lga_code']
source_item['xy'] = xys.get(source_id, None)
source_item['lga_version'] = '2020'
return source_data
def update_vic(self):
source = self._get_vic_source()
return self.target.update(rows=source, allow_deletes=False)
def full_update(self):
"""
Performs a full update from the source data, inculding adds, deletes and updates.
A combination of the date_code and postcode are used as the full identifier.
:return:
:rtype:
"""
unconsolidated = []
unconsolidated.extend(self._get_nsw_source())
unconsolidated.extend(self._get_vic_source())
consolidated = {}
for item in unconsolidated:
date_code = item['date_code']
lga_code = item['lga_code']
key = '{}_{}'.format(lga_code, date_code)
existing = consolidated.get(key)
if existing:
existing['cases'] += item['cases']
existing['total_cases'] += item['total_cases']
else:
consolidated[key] = copy.deepcopy(
item) # use a copy so original source not altered.
return self.target.update(rows=consolidated.values(),
allow_deletes=True)
def __init__(self, service_url=default_service_url):
self.target = DailyCovid19CasesByPostcode(service_url)
self.nsw_source = NswNotificationData()
self.geometry_source = POA2016.centroids()
class UpdateFromSourceTask(object):
"""
An update task encapsulating all of the required input parameters and structures.
Upon initial creation the defaults are set.
Defaults can be modified after creation using through the class properties.
"""
def __init__(self, service_url=default_service_url):
self.target = DailyCovid19CasesByPostcode(service_url)
self.nsw_source = NswNotificationData()
self.geometry_source = POA2016.centroids()
def full_update(self):
"""
Performs a full update from the source data, inculding adds, deletes and updates.
A combination of the date_code and postcode are used as the full identifier.
:return:
:rtype:
"""
# get the source data
source_data = self.nsw_source.source_data()
# sort rows by date and postcode.
sorted_data = sorted(
source_data,
key=lambda x:
(x[self.nsw_source.postcode_field], x[self.nsw_source.date_field]))
# get postcode xys
xys = da_arcpy.load_xy_geometries(
source=self.geometry_source._source,
id_field=self.geometry_source.poa_code_field)
# convert to input rows
totals = {}
wkg = {}
for row in sorted_data:
date_str = self.target.format_date_string(
date_string=row[self.nsw_source.date_field],
in_format=self.nsw_source.csv_date_format)
postcode = row[self.nsw_source.postcode_field]
if not postcode:
postcode = '0000'
total = totals.get(postcode, 0) + 1
totals[postcode] = total
key = '{}_{}'.format(postcode, date_str)
wkg_item = wkg.get(key, None)
if wkg_item:
wkg_item['cases'] += 1
wkg_item['total_cases'] = total
else:
xy = xys.get(postcode, None)
wkg_item = {
'postcode': postcode,
'date_code': date_str,
'cases': 1,
'total_cases': total,
'xy': xy
}
wkg[key] = wkg_item
return self.target.update(rows=wkg.values(), allow_deletes=True)
class UpdateFromSourceTask(object):
"""
An update task encapsulating all of the required input parameters and structures.
Upon initial creation the defaults are set.
Defaults can be modified after creation using through the class properties.
"""
def __init__(self, service_url=default_service_url):
self.target = TotalCovid19CasesByPostcode(service_url)
self.nsw_source = NswNotificationData()
self.geometry_source = POA2016.centroids()
def full_update(self):
"""
Performs a full update from the source data, inculding adds, deletes and updates.
A combination of the date_code and postcode are used as the full identifier.
:return:
:rtype:
"""
update_data = {}
self.append_nsw_cases_to_totals(update_data)
self.apply_xys(update_data)
return self.target.update(updates_items=update_data)
def append_nsw_cases_to_totals(self, postcode_items):
"""
Adds NSW cases to total_cases and updates date_of_last_case where case date is more recent than existing value.
Creates new postcode items where needed.
:param postcode_items: {postcode: {'postcode': str, 'total_cases': int, 'date_of_last_case': datetime}}
:type postcode_items: dict
:return: {postcode: {'postcode': str, 'total_cases': int, 'date_of_last_case': datetime}}
:rtype: dict
"""
# get the source data
source_data = self.nsw_source.source_data()
for case in source_data:
date_string = case[self.nsw_source.date_field]
date = datetime.datetime.strptime(date_string,
self.nsw_source.csv_date_format)
postcode = case[self.nsw_source.postcode_field]
postcode_item = postcode_items.get(postcode, None)
if postcode_item:
postcode_item['total_cases'] += 1
# if this case is more recent that the current date of last case, update the date of last case.
if postcode_item['date_of_last_case'] < date:
postcode_item['date_of_last_case'] = date
else:
postcode_items[postcode] = {
'postcode': postcode,
'total_cases': 1,
'date_of_last_case': date
}
return postcode_items
def apply_xys(self, postcode_items):
"""
Applys the xy geometries to each postcode using the geometry_source. If no geometry for the postcode is found, the output
'xy' value will be None for that item.
:param postcode_items: {postcode: {}}
:type postcode_items: dict
:return: {postcode: {'xy': {'x': float, 'y': float}}
:rtype: dict
"""
# get postcode xys
xys = da_arcpy.load_xy_geometries(
source=self.geometry_source._source,
id_field=self.geometry_source.poa_code_field)
for postcode, postcode_item in postcode_items.items():
postcode_item['xy'] = xys.get(postcode, None)
return postcode_items