def _get_from_path(self, path_data_page, date): r = [] with open(path_data_page, 'r', encoding='utf-8') as f: for item in csv.DictReader(f): r.append( DataPoint( region_schema=Schemas.POSTCODE, region_parent='AU-VIC', region_child=item['Postcode'], datatype=DataTypes.TOTAL, value=int(item['Confirmed cases (ever)'] or 0), date_updated=date, # FIXME!!!!! source_url=self.SOURCE_URL, source_id=self.SOURCE_ID)) r.append( DataPoint( region_schema=Schemas.POSTCODE, region_parent='AU-VIC', region_child=item['Postcode'], datatype=DataTypes.STATUS_ACTIVE, value=int(item['Active cases (current)'] or 0), date_updated=date, # FIXME!!!!! source_url=self.SOURCE_URL, source_id=self.SOURCE_ID)) return r
def _get_timeline(self): # {"UpdateDate":"14\/05\/2020 11:35", # "Source":"https:\/\/covid19.th-stat.com\/", # "DevBy":"https:\/\/www.kidkarnmai.com\/", # "SeverBy":"https:\/\/smilehost.asia\/", # "Data":[{ # "Date":"01\/01\/2020", # "NewConfirmed":0, # "NewRecovered":0, # "NewHospitalized":0, # "NewDeaths":0, # "Confirmed":0, # "Recovered":0, # "Hospitalized":0, # "Deaths":0 # }, ... r = [] text = self.get_text('timeline.json', include_revision=True) data = json.loads(text) for item in data['Data']: if not item['Date']: continue date = self.convert_date(item['Date'], formats=('%m/%d/%Y', )) r.append( DataPoint(region_schema=Schemas.ADMIN_0, region_parent='', region_child='TH', datatype=DataTypes.TOTAL, value=int(item['Confirmed']), date_updated=date, source_url=self.SOURCE_URL)) r.append( DataPoint(region_schema=Schemas.ADMIN_0, region_parent='', region_child='TH', datatype=DataTypes.STATUS_RECOVERED, value=int(item['Recovered']), date_updated=date, source_url=self.SOURCE_URL)) r.append( DataPoint(region_schema=Schemas.ADMIN_0, region_parent='', region_child='TH', datatype=DataTypes.STATUS_HOSPITALIZED, value=int(item['Hospitalized']), date_updated=date, source_url=self.SOURCE_URL)) r.append( DataPoint(region_schema=Schemas.ADMIN_0, region_parent='', region_child='TH', datatype=DataTypes.STATUS_DEATHS, value=int(item['Deaths']), date_updated=date, source_url=self.SOURCE_URL)) return r
def _get_total_cases_by_region(self, url, html): if url == self.STATS_BY_REGION_URL: tables = self._pq_contains( html, 'table', 'LGA Region', ignore_case=True) or [] du = self._get_date(url, html) r = [] for table in tables: for lga, num_cases in table[1]: lga = pq(lga).text().strip() if lga.lower() == 'total': # This value is very often out of date!!! ==================================================== if False: r.append( DataPoint(region_schema=Schemas.THS, region_parent='au-tas', region_child=pq(table[0][0][0]).text( ).strip().split(' - ')[-1].strip(), datatype=DataTypes.TOTAL, value=int( pq(num_cases).text().replace( ',', '').strip()), date_updated=du, source_url=url)) else: r.append( DataPoint(region_schema=Schemas.LGA, region_parent='au-tas', region_child=lga, datatype=DataTypes.TOTAL, value=int( pq(num_cases).text().replace( ',', '').strip()), date_updated=du, source_url=url)) return r else: table = self._pq_contains(html, 'table', 'Local Government Area', ignore_case=True) du = self._get_date(url, html) r = [] if table: for region_child, lga, num_cases in table[0][1]: r.append( DataPoint(region_schema=Schemas.LGA, region_parent='au-tas', region_child=pq(lga).text().strip(), datatype=DataTypes.TOTAL, value=int( pq(num_cases).text().replace( ',', '').strip()), date_updated=du, source_url=url)) return r
def _get_datapoints(self, path): date = path.name.split('-')[0] print(date) for path in path.iterdir(): print(path) with open(path, 'r', encoding='utf-8') as f: r = [] text = json.loads(f.read()) data = self.get_from_multipart(text, 'dataColumns') values_by_idx = self.get_recursively(data, 'dataColumns')[0]['dataValues'] lga_by_idx = self.get_recursively(data, 'dataColumns')[2]['dataValues'] active_idx = self.get_recursively(data, 'paneColumnsList')[0]['vizPaneColumns'][3]['aliasIndices'] total_idx = self.get_recursively(data, 'paneColumnsList')[0]['vizPaneColumns'][4]['aliasIndices'] for _active, lga in zip(active_idx, lga_by_idx[1:]): if _active in (-44, -70, -71): continue elif _active < 0: continue raise Exception(_active) r.append(DataPoint( region_schema=Schemas.LGA, region_parent='AU-SA', region_child=normalize_locality_name(lga), datatype=DataTypes.STATUS_ACTIVE, value=int(values_by_idx[_active]), date_updated=date, source_url=self.SOURCE_URL, source_id=self.SOURCE_ID )) for _total, lga in zip(total_idx, lga_by_idx[1:]): if _total == -44: continue elif _total < 0: raise Exception(_total) r.append(DataPoint( region_schema=Schemas.LGA, region_parent='AU-SA', region_child=normalize_locality_name(lga), datatype=DataTypes.TOTAL, value=int(values_by_idx[_total]), date_updated=date, source_url=self.SOURCE_URL, source_id=self.SOURCE_ID )) #print(values_by_idx) #print(lga_by_idx) #pprint(total_idx) return r
def _get_source_of_infection(self, updated_date, response_dict): # * Overseas acquired # * Cruise ship acquired (included in overseas acquired) # * Interstate acquired # * Locally acquired - contact of a confirmed case # * Locally acquired - contact not identified # * Under investigation # Normalise it with other states vic_norm_map = { 'Travel overseas': DataTypes.SOURCE_OVERSEAS, 'Contact with a confirmed case': DataTypes.SOURCE_CONFIRMED, 'Acquired in Australia, unknown source': DataTypes.SOURCE_COMMUNITY, 'Under investigation': DataTypes.SOURCE_UNDER_INVESTIGATION } output = [] data = response_dict['source_of_infection'][1] added = set() for source in data['result']['data']['dsr']['DS'][0]['PH'][0]['DM0']: output.append( DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-VIC', datatype=vic_norm_map[source['C'][0]], value=source['C'][1], date_updated=updated_date, source_url=self.SOURCE_URL, source_id=self.SOURCE_ID)) added.add(vic_norm_map[source['C'][0]]) for datatype in vic_norm_map.values(): if datatype in added: continue # Sometimes "under investigation" isn't provided, # but probably can assume at 0 for these days output.append( DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-VIC', datatype=datatype, value=0, date_updated=updated_date, source_url=self.SOURCE_URL, source_id=self.SOURCE_ID)) return output
def get_statistic(self): r = [] data = json.loads(self.get_text('statistic.json', include_revision=True)) by_region = Counter() for region_data in data: # {"dataSet":{"id":1,"code":"COVID19","name":"COVID19 statistics", # "shortName":"COVID19 stat","sourceUrl":"file:///tmp/covid19stat.txt", # "resourceUrl":null,"dataSetGroup":{"id":1,"name":"Default","pos":1, # "icon_resource":null},"pos":1,"isComparable":null,"delimiter":";"}, #print(region_data) for point_dict in region_data['points']: # {"abscissa":{"id":45862,"year":2020,"month":3,"day":10, # "name":"2020-03-10","date":"2020-03-10"},"ordinate":0.0} #print(point_dict['abscissa']['date']) if point_dict['ordinate'] is None: continue region_child = region_map[region_data['name'].lower().strip()] value = int(point_dict['ordinate']) date = self.convert_date(point_dict['abscissa']['date']) by_region[date, region_child] += value r.append(DataPoint( region_schema=Schemas.ADMIN_1, region_parent='RS', region_child=region_child, datatype=DataTypes.NEW, value=value, source_url=self.SOURCE_URL, date_updated=date )) cumulative = Counter() for (date, region_child), value in sorted(by_region.items()): cumulative[region_child] += value r.append(DataPoint( region_schema=Schemas.ADMIN_1, region_parent='RS', region_child=region_child, datatype=DataTypes.TOTAL, value=cumulative[region_child], source_url=self.SOURCE_URL, date_updated=date )) return r
def _get_total_male_female_breakdown(self, url, html): du = self._get_date(url, html) regex = compile( r'Total cases include ([0-9,]+) men and ([0-9,]+) women') match = regex.search(html) if match: men = int(match.group(1).replace(',', '')) women = int(match.group(2).replace(',', '')) men = DataPoint(region_schema=Schemas.ADMIN_1, region_parent='au', region_child='au-vic', date_updated=du, datatype=DataTypes.TOTAL_MALE, value=men, source_url=url) women = DataPoint(region_schema=Schemas.ADMIN_1, region_parent='au', region_child='au-vic', date_updated=du, datatype=DataTypes.TOTAL_FEMALE, value=women, source_url=url) return men, women else: men = self._extract_number_using_regex( compile('total[^0-9.]+?([0-9,]+) men'), html, region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-VIC', source_url=url, datatype=DataTypes.TOTAL_MALE, date_updated=du) women = self._extract_number_using_regex( compile('total[^0-9.]+?([0-9,]+) women'), html, region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-VIC', source_url=url, datatype=DataTypes.TOTAL_FEMALE, date_updated=du) if men is not None and women is not None: return men, women return None
def _get_total_cases_tested(self, url, html): neg_cases = self._extract_number_using_regex( # Seems the WA website's wording can change day-to-day compile(r'([0-9]+[0-9,]*?)' r'([^0-9]*?negative COVID-19 tests|' r'[^0-9]*?tested negative|' r'[^0-9]*?negative)'), html, region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-WA', source_url=url, datatype=DataTypes.TESTS_TOTAL, date_updated=self._get_date(url, html)) pos_cases = self._get_total_cases(url, html) if neg_cases and pos_cases: return DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-WA', datatype=neg_cases.datatype, value=neg_cases.value + pos_cases.value, date_updated=neg_cases.date_updated, source_url=neg_cases.source_url, text_match=(neg_cases.text_match, pos_cases.text_match)) return None
def _get_total_new_cases(self, href, html): c_html = word_to_number(html) if 'same total number as yesterday' in html: # https://www.dhhs.vic.gov.au/coronavirus-update-victoria-27-april-2020 return DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-VIC', datatype=DataTypes.NEW, value=0, date_updated=self._get_date(href, html), source_url=href, text_match='same total number as yesterday') return self._extract_number_using_regex( compile('increase of ([0-9,]+)'), c_html, region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-VIC', datatype=DataTypes.NEW, source_url=href, date_updated=self._get_date( href, html)) or self._extract_number_using_regex( compile('([0-9,]+) new cases'), c_html, region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-VIC', datatype=DataTypes.NEW, source_url=href, date_updated=self._get_date(href, html))
def _get_recovered_sum(self): r = [] base_dir = self.get_path_in_dir('') for date in self.iter_nonempty_dirs(base_dir): path = f'{base_dir}/{date}/is_index.html' with open(path, 'rb') as f: data = f.read() data = data.decode('utf-8') # TODO: There are quite a few more stats!! regional_stats = data.split('[[[null,{"font-weight":"700","value":"Infections"},' '{"font-weight":"700","value":"Quarantine"}],')[1].split(']]],')[0] #print(regional_stats) regional_stats = json.loads(f'[{regional_stats}]]') for region, infections_dict, quarantine_dict in regional_stats: region = place_map[region] r.append(DataPoint( region_schema=Schemas.ADMIN_1, region_parent='IS', region_child=region, datatype=DataTypes.TOTAL, # This changed to be an int from a dict on 9 Jun value=int(infections_dict['value']) if isinstance(infections_dict, dict) else int(infections_dict), date_updated=date, source_url=self.SOURCE_URL )) return r
def _get_total_cases(self, href, html): du = self._get_date(href, html) if href in (self.STATS_BY_REGION_URL, self.STATS_BY_REGION_URL_2): # New format as of 22 April cases = pq(html)('.qh-fact-wrapper .cases span') if cases: return self._extract_number_using_regex( compile('([0-9,]+)'), pq(cases[0]).text().strip(), region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-QLD', datatype=DataTypes.TOTAL, date_updated=du, source_url=href) else: return None # Use new format from the table if possible totals_dict = self.__get_totals_from_table(html) if totals_dict: return DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-QLD', datatype=DataTypes.TOTAL, value=totals_dict['total'], date_updated=du, source_url=href, text_match=None) c_html = word_to_number(html) return self._extract_number_using_regex( (compile('state total to ([0-9,]+)'), compile('total of ([0-9,]+) (?:people|person)')), c_html, region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-QLD', source_url=href, datatype=DataTypes.TOTAL, date_updated=du ) or self._extract_number_using_regex( compile( # Total number changed from being enclosed in a <strong> # tag to a <b> tag, so changed to be as broad as NSW # <strong>Total</strong></td> # <td headers="table59454r1c2"><b>37,334‬</b></td> r'<td[^>]*?>(?:<[^</>]+>)?Total(?:</[^<>]+>)?</td>' r'[^<]*?<td[^>]*?>.*?([0-9,]+).*?</td>', MULTILINE | DOTALL), c_html, region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-QLD', source_url=href, datatype=DataTypes.TOTAL, date_updated=du)
def _get_all_lga_datapoints(self, date): r = [] current_date = None by_agegroup = ExpiringCounter() with open(get_data_dir() / 'vic' / 'csv_data' / date / 'all_lga.csv', 'r', encoding='utf-8') as f: for row in sorted(csv.DictReader(f), key=lambda x: x['diagnosis_date']) + \ [{'diagnosis_date': '1111-01-01', 'Localgovernmentarea': None}]: date_updated = self.convert_date(row['diagnosis_date']) if current_date != date_updated: if current_date is not None: for lga, value in by_agegroup.items(): r.append( DataPoint(region_schema=Schemas.LGA, region_parent='AU-VIC', region_child=normalize_locality_name( lga.split('(')[0].strip()), datatype=DataTypes.TOTAL, value=int(value), date_updated=current_date, source_url=self.SOURCE_URL, source_id=self.SOURCE_ID)) current_date = date_updated if row['Localgovernmentarea']: by_agegroup[row['Localgovernmentarea'].strip('_')] += 1 return r
def _get_agegroup_datapoints(self, date): r = [] current_date = None by_agegroup = Counter() with open(get_data_dir() / 'vic' / 'csv_data' / date / 'agegroup.csv', 'r', encoding='utf-8') as f: for row in sorted(csv.DictReader(f), key=lambda x: x['diagnosis_date']) + \ [{'diagnosis_date': '1111-01-01', 'agegroup': None}]: assert len( row['diagnosis_date']) in (9, 10), row['diagnosis_date'] date_updated = self.convert_date(row['diagnosis_date']) if current_date != date_updated: if current_date is not None: for agerange, value in by_agegroup.items(): r.append( DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-VIC', datatype=DataTypes.TOTAL, agerange=agerange, value=int(value), date_updated=current_date, source_url=self.SOURCE_URL, source_id=self.SOURCE_ID)) current_date = date_updated if row['agegroup']: by_agegroup[row['agegroup'].strip('_')] += 1 return r
def __get_tests_datapoints(self, SOURCE_URL, path_tests): r = DataPointMerger() with open(path_tests, 'r', encoding='utf-8') as f: for item in json.loads(f.read())['data']: try: item['POA_NAME16'] = str(int(float(item['POA_NAME16']))) except ValueError: pass date = self.__get_partial_date(path_tests, item['Date']) number = int(item['Number']) # recent = item['Recent'] # TODO: ADD ME!!! ======================================================== postcode = item['POA_NAME16'] if item['POA_NAME16'] else 'Unknown' r.append(DataPoint( region_schema=Schemas.POSTCODE, region_parent='AU-NSW', region_child=postcode, datatype=DataTypes.TESTS_TOTAL, value=number, date_updated=date, source_url=SOURCE_URL, source_id=self.SOURCE_ID )) return r
def _get_total_source_of_infection(self, url, html): norm_map = { 'Locally Acquired—close contact with confirmed case': DataTypes.SOURCE_CONFIRMED, 'Locally Acquired—no known contact': DataTypes.SOURCE_COMMUNITY, 'Interstate acquired': DataTypes.SOURCE_INTERSTATE, 'Overseas acquired': DataTypes.SOURCE_OVERSEAS, 'Under investigation': DataTypes.SOURCE_UNDER_INVESTIGATION, } du = self._get_date(url, html) if url == self.STATS_BY_REGION_URL_2: table = pq(html)('#QLD_Cases_Sources_Of_Infection')[0] #print(pq(table).html()) r = [] for header, value in table[0]: header = pq(header).text().strip() if header in ('Confirmed cases', 'Total cases'): continue value = pq(value).text().strip() r.append( DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-QLD', datatype=norm_map[header], value=int(value.replace(',', '')), date_updated=du, source_url=url)) return r else: return []
def _get_lga_datapoints(self, date): # LGA lga_pid population active cases rate new band LGADisplay data_date # Alpine (S) VIC242 12814 0 1 0 0 0 Alpine 29/08/2020 # Ararat (RC) VIC220 11845 1 7 8.4 0 1 Ararat 29/08/2020 # Ballarat (C) VIC241 109505 6 61 5.5 0 1 Ballarat 29/08/2020 # Banyule (C) VIC188 131631 30 437 22.8 0 2 Banyule 29/08/2020 # Bass Coast (S) VIC173 36320 0 11 0 0 0 Bass Coast 29/08/2020 # Baw Baw (S) VIC194 53396 1 15 1.9 0 1 Baw Baw 29/08/2020 # Bayside (C) VIC182 106862 72 227 67.4 6 3 Bayside 29/08/2020 # Benalla (RC) VIC199 14037 0 3 0 0 0 Benalla 29/08/2020 r = [] print("LGA:", get_data_dir() / 'vic' / 'csv_data' / date) with open(get_data_dir() / 'vic' / 'csv_data' / date / 'lga.json', 'r', encoding='utf-8') as f: for row in csv.DictReader(f): #print(row) date_updated = self.convert_date(row['data_date']) for datatype, value in ((DataTypes.STATUS_ACTIVE, row['active']), (DataTypes.TOTAL, row['cases'])): r.append( DataPoint(region_schema=Schemas.LGA, region_parent='AU-VIC', region_child=normalize_locality_name( row['LGA'].split('(')[0].strip()), datatype=datatype, value=int(value), date_updated=date_updated, source_url=self.SOURCE_URL, source_id=self.SOURCE_ID)) return r
def __get_merged_datapoint(self, datapoint): """ Find+remove the previous datapoint (if it exists); return a new datapoint with both values added """ unique_key = self.__get_unique_key(datapoint) if unique_key in self.__datapoint_indexes: replace_index = self.__datapoint_indexes[unique_key] i = self[replace_index] r = DataPoint(region_schema=datapoint.region_schema, region_parent=datapoint.region_parent, region_child=datapoint.region_child, date_updated=datapoint.date_updated, datatype=datapoint.datatype, agerange=datapoint.agerange, value=datapoint.value + i.value, source_url=datapoint.source_url or i.source_url, text_match=datapoint.text_match or i.text_match, source_id=datapoint.source_id) else: replace_index = None r = datapoint return replace_index, r
def _get_regions(self, updated_date, response_dict): output = [] data = response_dict['regions'][1] previous_value = None for region_child in data['result']['data']['dsr']['DS'][0]['PH'][0][ 'DM0']: value, previous_value = self.process_powerbi_value( region_child, previous_value, data) if value[0] is None: continue region_string = value[0].split('(')[0].strip() output.append( DataPoint(region_schema=Schemas.LGA, region_parent='au-vic', region_child=region_string, datatype=DataTypes.TOTAL, value=value[1], date_updated=updated_date, source_url=self.SOURCE_URL, source_id=self.SOURCE_ID)) previous_value = value # print(output[-1]) self.totals_dict[region_string] = value[1] return output
def _get_total_age_breakdown(self, href, html): if href == self.STATS_BY_REGION_URL_2: r = [] table = pq(html)('#QLD_CasesByAgeAndGender')[0][1] du = self._get_date(href, html) for tr in table[1:]: age_group = pq(tr[0]).text().strip() female = int(pq(tr[1]).text().replace(',', '')) male = int(pq(tr[2]).text().replace(',', '')) total = int(pq(tr[3]).text().replace(',', '')) for datatype, value in ((DataTypes.TOTAL_FEMALE, female), (DataTypes.TOTAL_MALE, male), (DataTypes.TOTAL, total)): if value is None: continue r.append( DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-QLD', datatype=datatype, agerange=age_group, value=value, date_updated=du, source_url=href)) return r
def _get_total_age_breakdown(self, href, html): table = self._pq_contains(html, 'table', 'By age group', ignore_case=True) if not table: return # WARNING!!! ======================================================================================= du = self._get_date(href, html) table = table[0] tbody = pq(table)('tbody')[0] tr = tbody[1] ages = [ int(i.replace(',', '').strip()) for i in pq(tr).text().split('\n') ] ages = { '0-29': ages[0], '30-39': ages[1], '40-49': ages[2], '50-59': ages[3], '60-69': ages[4], '70+': ages[5] } r = [] for k, v in ages.items(): r.append( DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-ACT', datatype=DataTypes.TOTAL, agerange=k, value=v, date_updated=du, source_url=href)) return r
def _get_postcode_datapoints(self, date): # postcode population active cases rate new band data_date # 3000 37979 18 119 47.4 0 2 29/08/2020 # 3001 0 0 1 0 0 0 29/08/2020 # 3002 4957 2 14 40.3 0 2 29/08/2020 # 3003 5516 3 36 54.4 0 3 29/08/2020 # 3004 9311 6 63 64.4 2 3 29/08/2020 # 3005 523 0 0 0 0 0 29/08/2020 # 3006 18811 1 64 5.3 0 1 29/08/2020 # 3008 10438 2 49 19.2 0 1 29/08/2020 # 3010 1595 0 0 0 0 0 29/08/2020 # 3011 21464 36 164 167.7 2 4 29/08/2020 r = [] print("PostCode:", get_data_dir() / 'vic' / 'csv_data' / date) with open(get_data_dir() / 'vic' / 'csv_data' / date / 'postcode.json', 'r', encoding='utf-8') as f: for row in csv.DictReader(f): date_updated = self.convert_date(row['data_date']) for datatype, value in ((DataTypes.STATUS_ACTIVE, row['active']), (DataTypes.TOTAL, row['cases'])): r.append( DataPoint(region_schema=Schemas.POSTCODE, region_parent='AU-VIC', region_child=row['postcode'], datatype=datatype, value=int(value), date_updated=date_updated, source_url=self.SOURCE_URL, source_id=self.SOURCE_ID)) return r
def _get_gender_balance_data(self, updated_date, response_dict): r = [] try: data = response_dict['gender_balance'][1] except KeyError: return [] # WARNING!!! ================================================================================== # WARNING: This sometimes has another query before it!!! ======================================================= try: m_f = data['result']['data']['dsr']['DS'][0]['PH'][0]['DM0'] assert m_f[0]['C'][0] in ('Males', 'Male') assert m_f[1]['C'][0] in ('Females', 'Female') except: m_f = data['result']['data']['dsr']['DS'][0]['PH'][0]['DM0'] assert m_f[0]['C'][0] in ('Males', 'Male') assert m_f[1]['C'][0] in ('Females', 'Female') male = m_f[0]['C'][1] try: female = m_f[1]['C'][1] except IndexError: assert m_f[1]['R'] female = male r.append(DataPoint( region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-ACT', datatype=DataTypes.TOTAL_MALE, value=self._to_int(male), date_updated=updated_date, source_url=self.source_url, source_id=self.SOURCE_ID )) r.append(DataPoint( region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-ACT', datatype=DataTypes.TOTAL_FEMALE, value=self._to_int(female), date_updated=updated_date, source_url=self.source_url, source_id=self.SOURCE_ID )) return r
def __postcode_datapoints_to_lga(self, SOURCE_URL, postcode_to_lga, r, source_id): # Convert postcode to LGA where possible new_r = DataPointMerger() added_to_lga = set() processed_postcode = set() mapping = Counter() for datapoint in sorted(r, key=lambda i: i.date_updated): if datapoint.region_schema == Schemas.LGA: added_to_lga.add(( datapoint.region_child, datapoint.datatype )) continue elif datapoint.region_schema != Schemas.POSTCODE: continue elif datapoint.region_child in postcode_to_lga: lga = postcode_to_lga[datapoint.region_child] else: lga = 'unknown' if datapoint.region_child != 'unknown': print("NOT FOUND:", datapoint.region_child) # continue # WARNINIG!!! ================================================================================ if (datapoint.region_child, datapoint.datatype, datapoint.date_updated) in processed_postcode: #print("IGNORING DOUBLE-UP:", datapoint) continue processed_postcode.add((datapoint.region_child, datapoint.datatype, datapoint.date_updated)) #if lga == 'cumberland': # print('USING:', datapoint) mapping[ lga, datapoint.datatype, datapoint.date_updated ] += datapoint.value new_r.extend(r) for (lga, datatype, date_updated), value in mapping.items(): if (lga, datatype) in added_to_lga: # Don't add to LGA if available using direct data! continue new_r.append(DataPoint( region_schema=Schemas.LGA, region_parent='AU-NSW', region_child=lga, datatype=datatype, value=value, date_updated=date_updated, source_url=SOURCE_URL, source_id=source_id )) return new_r
def _get_total_age_breakdown(self, href, html): # TODO: TRANSITION TO https://data.nsw.gov.au/nsw-covid-19-data !! ============================================= if '20200316_02.aspx' in href: # HACK: The very first entry was in a different format with percentages # Maybe I could fix this later, but not sure it's worth it return None r = [] table = self._pq_contains( html, 'table', 'Age Group', ignore_case=True ) if not table: return None table = table[0] du = self._get_date(href, html) for age_group in ( '0-9', '10-19', '20-29', '30-39', '40-49', '50-59', '60-69', '70-79', '80-89', '90-100' ): tds = self._pq_contains(table, 'tr', age_group) if not tds: continue tds = tds[0] female = int(pq(tds[1]).text().strip() or 0) male = int(pq(tds[2]).text().strip() or 0) total = int(pq(tds[3]).text().replace(' ', '').strip() or 0) for datatype, value in ( (DataTypes.TOTAL_FEMALE, female), (DataTypes.TOTAL_MALE, male), (DataTypes.TOTAL, total) ): if value is None: continue r.append(DataPoint( region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-NSW', datatype=datatype, agerange=age_group, value=value, date_updated=du, source_url=href )) return r
def _extract_number_using_regex(self, regex, s, source_url, datatype, date_updated, agerange=None, region_parent=None, region_child=None, region_schema=Schemas.ADMIN_1): """ Convenience function for removing numeral grouping X,XXX and returning a number based on a match from re.compile() instance `regex` Multiple regexes can be specified for `regex`, in which case the first match will be returned """ # assert region_parent assert region_child if isinstance(regex, (list, tuple)): for i_regex in regex: dp = self._extract_number_using_regex( i_regex, s, source_url, datatype, date_updated, agerange, region_parent, region_child, region_schema ) if dp: return dp return None match = regex.search(s) # print(regex, match) if match: num = match.group(1) num = num.replace(',', '') if num.isdecimal(): #print(f" Found Match: {match.group()}") num = int(num) if date_updated is None: date_updated = self._todays_date() return DataPoint( region_schema=region_schema, region_parent=region_parent, region_child=region_child, datatype=datatype, agerange=agerange, value=num, date_updated=date_updated, source_url=source_url, text_match=s[ max(0, match.start(1)-40): min(len(s), match.end(1)+40) ] ) return None
def _get_total_source_of_infection(self, url, html): # NOTE: there are also stats at # https://www.covid19.act.gov.au/updates/confirmed-case-information # but they're in a different format - # not sure it's worth supporting them # Normalise it with other states act_norm_map = { 'Overseas acquired': DataTypes.SOURCE_OVERSEAS, 'Cruise ship acquired': DataTypes.SOURCE_CRUISE_SHIP, 'Interstate acquired': DataTypes.SOURCE_INTERSTATE, 'Contact of a confirmed ACT case': DataTypes.SOURCE_CONFIRMED, 'Unknown or local transmission': DataTypes.SOURCE_COMMUNITY, 'Under investigation': DataTypes.SOURCE_UNDER_INVESTIGATION, } du = self._get_date(url, html) r = [] for re_text in ( r'<tr[^>]*><td[^>]*><p[^>]*>Overseas acquired</p></td>' r'<td[^>]*><p[^>]*>(?P<Overseas_acquired>[0-9,]+)</p></td></tr>', # Cruise ship-acquired was only added around 6 April r'<tr[^>]*><td[^>]*><p[^>]*>Cruise ship acquired</p></td>' r'<td[^>]*><p[^>]*>(?P<Cruise_ship_acquired>[0-9,]+) of the [0-9,]+</p></td></tr>', r'<tr[^>]*><td[^>]*><p[^>]*>Interstate acquired</p></td>' r'<td[^>]*><p[^>]*>(?P<Interstate_acquired>[0-9,]+)</p></td></tr>', r'<tr[^>]*><td[^>]*><p[^>]*>Contact of a confirmed ACT case</p></td>' r'<td[^>]*><p[^>]*>(?P<Contact_of_a_confirmed_ACT_case>[0-9,]+)</p></td></tr>', r'<tr[^>]*><td[^>]*><p[^>]*>Unknown / local transmission</p></td>' r'<td[^>]*><p[^>]*>(?P<Unknown_or_local_transmission>[0-9,]+)</p></td></tr>', r'<tr[^>]*><td[^>]*><p[^>]*>Under investigation</p></td>' r'<td[^>]*><p[^>]*>(?P<Under_investigation>[0-9,]+)</p></td></tr>' ): re_soi = compile(re_text, IGNORECASE) match = re_soi.search(html) if match: gd = match.groupdict() for k, v in gd.items(): if v is None: continue r.append( DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-ACT', datatype=act_norm_map[k.replace('_', ' ')], value=int(v.replace(',', '')), date_updated=du, source_url=url)) return r or None
def _get_datapoints(date, path): r = [] with open(path, 'r', encoding='utf-8') as f: data = json.loads(f.read()) if isinstance(data, list): for row in data: for datatype, value in ((DataTypes.STATUS_ACTIVE, row['activedisp']), (DataTypes.TOTAL, row['cases'])): if value == 'Five or fewer active cases': continue r.append( DataPoint(region_schema=Schemas.POSTCODE, region_parent='AU-VIC', region_child=str(row['postcode']), datatype=datatype, value=int(value), date_updated=date, source_url=SOURCE_URL, source_id=SOURCE_ID)) else: for row in data['rows']: for datatype, value in ((DataTypes.STATUS_ACTIVE, row['active']), (DataTypes.TOTAL, row['total'])): r.append( DataPoint(region_schema=Schemas.POSTCODE, region_parent='AU-VIC', region_child=str(row['postcode']), datatype=datatype, value=int(value), date_updated=date, source_url=SOURCE_URL, source_id=SOURCE_ID)) return r
def add_new_datapoints_from_total(self, source_id, new_datatype, total_datatype): print("Adding new datapoints from totals:", source_id) new_datapoints = self.datapoints_db.select_many( source_id=['=?', [source_id]], datatype=['=?', [new_datatype]]) total_datapoints = self.datapoints_db.select_many( source_id=['=?', [source_id]], datatype=['=?', [total_datatype]]) n = {} t = {} for new_datapoint in new_datapoints: n[new_datapoint.date_updated, new_datapoint.region_schema, new_datapoint.region_parent, new_datapoint.region_child, new_datapoint.agerange] = new_datapoint for total_datapoint in total_datapoints: t[total_datapoint.date_updated, total_datapoint.region_schema, total_datapoint.region_parent, total_datapoint.region_child, total_datapoint.agerange] = total_datapoint append_datapoints = [] for k, total_datapoint in t.items(): if k in n: # Already have a new datapoint for this, so don't add! continue day_before = date_fns.apply_timedelta(total_datapoint.date_updated, days=-1) k_pd = (day_before, ) + k[1:] # previous day if not k_pd in t: continue total_datapoint_pd = t[k_pd] append_datapoints.append( DataPoint(region_schema=total_datapoint.region_schema, region_parent=total_datapoint.region_parent, region_child=total_datapoint.region_child, date_updated=total_datapoint.date_updated, datatype=new_datatype, agerange=total_datapoint.agerange, value=total_datapoint.value - total_datapoint_pd.value, source_url='DERIVED')) self.datapoints_db.extend(source_id, append_datapoints, is_derived=True)
def _get_total_source_of_infection(self, url, html): """ Source Cases Overseas acquired 252 Locally acquired (close contact of a confirmed case) 78 Locally acquired (Interstate travel) 7 Locally acquired (contact not identified) 3 Under investigation 27 TOTAL 367 """ html = html.replace(' ', ' ') r = [] du = None # Normalise it with other states sa_norm_map = { 'Overseas acquired': DataTypes.SOURCE_OVERSEAS, 'Locally acquired (Interstate travel)': DataTypes.SOURCE_INTERSTATE, 'Locally acquired (close contact of a confirmed case)': DataTypes.SOURCE_CONFIRMED, 'Locally acquired (contact not identified)': DataTypes.SOURCE_COMMUNITY, 'Under investigation': DataTypes.SOURCE_UNDER_INVESTIGATION } for k in ('Overseas acquired', 'Locally acquired (close contact of a confirmed case)', 'Locally acquired (Interstate travel)', 'Locally acquired (contact not identified)', 'Under investigation'): tr = self._pq_contains(html, 'tr', k, ignore_case=True) if not tr: continue if du is None: du = self._get_date(url, html) tr = tr[0] c_icu = int(pq(tr[1]).text().strip()) r.append( DataPoint(region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-SA', datatype=sa_norm_map[k], value=c_icu, date_updated=du, source_url=url)) return r or None
def _get_recovered_data(self, updated_date, response_dict): r = [] data = response_dict['recovered'][1] recovered = data['result']['data']['dsr']['DS'][0]['PH'][0]['DM0'][0]['M0'] r.append(DataPoint( region_schema=Schemas.ADMIN_1, region_parent='AU', region_child='AU-ACT', datatype=DataTypes.STATUS_RECOVERED, value=self._to_int(recovered), date_updated=updated_date, source_url=self.source_url, source_id=self.SOURCE_ID )) return r