def slack_county_of_interest(self, record, record_last_week, channel):
cases_weekly = record.cumulative_count - record_last_week.cumulative_count
deaths_weekly = record.cumulative_deaths - record_last_week.cumulative_deaths
msg = '*{} County, {}*\n'.format(
record.county.name, record.scrape_date.strftime('%B %d, %Y'))
msg += f'*{record.cumulative_count:,}* cases total (change of *{self.change_sign(record.daily_total_cases)}* today, {self.change_sign(cases_weekly)} last 7 days)\n'
msg += f'*{record.cumulative_deaths:,}* deaths total (change of *{self.change_sign(record.daily_deaths)}* today, {self.change_sign(deaths_weekly)} last 7 days))\n'
msg += '\n\n\n'
slack_latest(msg, channel)
def get_critical_supplies(self, session, sessionid):
print("Caching critical supplies data...")
r = session.get(self.SUPPLIES_CSV.format(sessionid=sessionid))
if r.status_code == requests.codes.ok:
# print(r.text)
# Save a copy of CSV
now = datetime.datetime.now().strftime('%Y-%m-%d_%H%M')
outpath = os.path.join(settings.BASE_DIR, 'exports', 'dashboard',
'supplies_{}.csv').format(now)
with open(outpath, 'w') as csv_file:
csv_file.write(r.text)
csv_file.close()
else:
slack_latest(
'WARNING: Critical supplies CSV scraper error. Not proceeding.',
'#robot-dojo')
def get_hospital_capacities(self, session, sessionid):
print("Caching hospitalization data...")
r = session.get(self.HOSP_CAPACITY_CSV.format(sessionid=sessionid))
if r.status_code == requests.codes.ok:
# print(r.text)
# Save a copy of CSV
now = datetime.datetime.now().strftime('%Y-%m-%d_%H%M')
outpath = os.path.join(settings.BASE_DIR, 'exports', 'dashboard',
'hospital_capacity_{}.csv').format(now)
with open(outpath, 'w') as csv_file:
csv_file.write(r.text)
csv_file.close()
else:
slack_latest(
'WARNING: Hospital capacity CSV scraper error. Not proceeding.',
'#robot-dojo')
def get_csv(self):
for f in self.FILENAMES:
r = requests.get(f['orig_file'])
if r.status_code == requests.codes.ok:
# Save a copy of CSV
now = datetime.datetime.now().strftime('%Y-%m-%d_%H%M')
outpath = os.path.join(settings.BASE_DIR, 'exports',
'dashboard',
f['download_base_name']).format(now)
with open(outpath, 'w') as csv_file:
csv_file.write(r.text)
csv_file.close()
else:
slack_latest(
'WARNING: Dashboard CSV scraper error.: {}'.format(
f['orig_file']), '#robot-dojo')
def handle(self, *args, **options):
html = get_situation_page_content()
if not html:
slack_latest(
"COVID scraper ERROR: update_mn_county_data.py can't find page HTML. Not proceeding.",
'#robot-dojo')
else:
soup = BeautifulSoup(html, 'html.parser')
bool_updated_today, update_date = updated_today(soup)
if bool_updated_today:
print('Updated today')
county_data = self.get_county_data(soup)
if len(county_data) > 0:
bool_any_changes = self.update_county_records(
county_data, update_date)
if bool_any_changes:
slack_latest('*COVID daily update*\n\n',
'#duluth_live')
for county in [
'St. Louis', 'Carlton', 'Itasca', 'Lake',
'Cook'
]:
record = CountyTestDate.objects.get(
county__name=county,
scrape_date=datetime.date.today())
record_last_week = CountyTestDate.objects.get(
county__name=county,
scrape_date=datetime.date.today() -
datetime.timedelta(days=7))
self.slack_county_of_interest(
record, record_last_week, '#duluth_live')
slack_latest('*COVID daily update*\n\n',
'#stcloud_live')
for county in ['Stearns', 'Benton', 'Sherburne']:
record = CountyTestDate.objects.get(
county__name=county,
scrape_date=datetime.date.today())
record_last_week = CountyTestDate.objects.get(
county__name=county,
scrape_date=datetime.date.today() -
datetime.timedelta(days=7))
self.slack_county_of_interest(
record, record_last_week, '#stcloud_live')
else:
slack_latest(
'COVID scraper warning: No county records found.',
'#robot-dojo')
else:
print('No update yet today')
def handle(self, *args, **options):
try:
s = requests.Session()
# r = s.get('https://mn.gov/covid19/assets/HospitalCapacity_HistoricCSV_tcm1148-449110.csv', headers=self.HEADERS)
r = s.get(
'https://mn.gov/covid19/assets/MNTRAC_ICU_NonICU_BedAvailability_IdentifiedSurge_CSV_tcm1148-455098.csv',
headers=self.HEADERS)
csv_obj = csv.DictReader(io.StringIO(r.text))
capacity_raw_df = pd.DataFrame(csv_obj)
capacity_raw_df['Value_NUMBER'] = pd.to_numeric(
capacity_raw_df['Value_NUMBER'])
capacity_raw_df_statewide = capacity_raw_df[
capacity_raw_df['GeographicLevel'] == 'State']
print(capacity_raw_df_statewide)
# capacity_raw_df = capacity_raw_df[~capacity_raw_df['Detail4'].isin(['On back order', 'In warehouse'])]
# capacity_pivot = self.reshape_capacities(capacity_raw_df)
# counts_pivot = self.reshape_counts(capacity_raw_df)
#
# merged_df = counts_pivot.merge(capacity_pivot, how="outer", on="data_date")
# merged_df['data_date'] = pd.to_datetime(merged_df['data_date'])
# merged_df = merged_df.sort_values('data_date')
#
# merged_df['icu_total'] = merged_df['icu_covid'] + merged_df['icu_other']
# merged_df['covid_total'] = merged_df['icu_covid'] + merged_df['non_icu_covid']
#
# merged_df['icu_pct_cap'] = round(merged_df['icu_total'] / merged_df['icu_cap'], 3)
# merged_df['vent_pct_cap'] = round(merged_df['vents'] / merged_df['vent_cap'], 3)
#
# merged_df.to_csv(self.CLEAN_OUTPATH, index=False)
except Exception as e:
# raise
slack_latest(
'WARNING: Hospital capacity CSV scraper error... \n{}'.format(
e), '#robot-dojo')
def handle(self, *args, **options):
html = get_situation_page_content()
if not html:
slack_latest(
"COVID scraper ERROR: update_mn_age_data.py can't find page HTML. Not proceeding.",
'#robot-dojo')
else:
soup = BeautifulSoup(html, 'html.parser')
age_data = self.get_age_data(soup)
if len(age_data) > 0:
age_msg_output = self.update_age_records(age_data)
slack_latest(age_msg_output, '#robot-dojo')
else:
slack_latest('COVID scraper warning: No age records found.',
'#robot-dojo')
def handle(self, *args, **options):
html = get_situation_page_content()
if not html:
slack_latest('WARNING: Scraper error. Not proceeding.',
'#robot-dojo')
else:
previous_statewide_cases = StatewideTotalDate.objects.order_by(
'-scrape_date').first().cumulative_positive_tests
soup = BeautifulSoup(html, 'html.parser')
bool_updated_today, update_date = updated_today(soup)
print(update_date)
if bool_updated_today:
print('Updated today')
statewide_data = self.get_statewide_data(soup)
statewide_msg_output = self.update_statewide_records(
statewide_data, update_date)
self.get_statewide_cases_timeseries(soup, update_date)
test_msg_output = self.get_statewide_tests_timeseries(
soup, update_date)
death_msg_output = self.get_statewide_deaths_timeseries(
soup, update_date)
total_hospitalizations = self.get_statewide_hospitalizations_timeseries(
soup, update_date)
if statewide_data[
'cumulative_positive_tests'] != previous_statewide_cases:
slack_latest(
statewide_msg_output + death_msg_output +
test_msg_output, '#virus')
else:
# slack_latest(statewide_msg_output + death_msg_output + test_msg_output, '#virus') # Force output anyway
slack_latest('COVID scraper update: No changes detected.',
'#robot-dojo')
else:
print('No update yet today')
def handle(self, *args, **options):
# archive_list = [
# 'http://static.startribune.com.s3.amazonaws.com/news/projects/all/2021-covid-scraper/raw/html/situation_2021-03-26_1103.html',
# 'http://static.startribune.com.s3.amazonaws.com/news/projects/all/2021-covid-scraper/raw/html/situation_2021-03-27_1103.html',
# 'http://static.startribune.com.s3.amazonaws.com/news/projects/all/2021-covid-scraper/raw/html/situation_2021-03-28_1103.html'
# ]
# for url in archive_list:
# html = get_situation_page_content(url) # TEMP MANUAL OVERRIDE
html = get_situation_page_content() # TEMP MANUAL OVERRIDE
if not html:
slack_latest(
"COVID scraper ERROR: update_mn_recent_deaths.py can't find page HTML. Not proceeding.",
'#robot-dojo')
else:
soup = BeautifulSoup(html, 'html.parser')
# bool_updated_today, update_date = updated_today(soup, True) # TEMP: MANUAL OVERRIDE
bool_updated_today, update_date = updated_today(
soup) # TEMP: MANUAL OVERRIDE
if bool_updated_today:
print('Updated today')
recent_deaths_data = self.get_recent_deaths_data(
soup, update_date)
if recent_deaths_data == None:
slack_latest(
'COVID scraper warning: No recent deaths records found.',
'#robot-dojo')
elif type(recent_deaths_data
) == list and len(recent_deaths_data) > 0:
deaths_msg_output = self.load_recent_deaths(
recent_deaths_data, update_date)
# slack_latest(deaths_msg_output, '#robot-dojo')
else:
slack_latest('COVID scraper OK: Seems to be 0-death day.',
'#robot-dojo')
else:
print('No update yet today')
def update_statewide_records(self, statewide_data, update_date):
yesterday = update_date - timedelta(days=1)
yesterday_results = StatewideTotalDate.objects.get(
scrape_date=yesterday)
today = datetime.date.today()
total_statewide_tests = statewide_data['total_statewide_tests']
cases_daily_change = statewide_data[
'cumulative_positive_tests'] - yesterday_results.cumulative_positive_tests
deaths_daily_change = statewide_data[
'cumulative_statewide_deaths'] - yesterday_results.cumulative_statewide_deaths
hospitalized_total_daily_change = statewide_data[
'cumulative_hospitalized'] - yesterday_results.cumulative_hospitalized
icu_total_daily_change = statewide_data[
'cumulative_icu'] - yesterday_results.cumulative_icu
base_record_obj = {
'cumulative_positive_tests':
statewide_data['cumulative_positive_tests'],
'cases_daily_change':
cases_daily_change,
'cases_newly_reported':
statewide_data['cases_newly_reported'],
'removed_cases':
statewide_data['removed_cases'],
'deaths_daily_change':
deaths_daily_change,
'cumulative_completed_tests':
total_statewide_tests,
'cumulative_hospitalized':
statewide_data['cumulative_hospitalized'],
'hospitalized_total_daily_change':
hospitalized_total_daily_change,
'cumulative_icu':
statewide_data['cumulative_icu'],
'icu_total_daily_change':
icu_total_daily_change,
'cumulative_statewide_deaths':
statewide_data['cumulative_statewide_deaths'],
'cumulative_statewide_recoveries':
statewide_data['cumulative_statewide_recoveries'],
'confirmed_cases_newly_reported':
statewide_data['confirmed_cases_newly_reported'],
'probable_cases_newly_reported':
statewide_data['probable_cases_newly_reported'],
'cumulative_confirmed_cases':
statewide_data['cumulative_confirmed_cases'],
'cumulative_probable_cases':
statewide_data['cumulative_probable_cases'],
'cumulative_pcr_tests':
statewide_data['cumulative_pcr_tests'],
'cumulative_antigen_tests':
statewide_data['cumulative_antigen_tests'],
'cumulative_confirmed_statewide_deaths':
statewide_data['cumulative_confirmed_statewide_deaths'],
'cumulative_probable_statewide_deaths':
statewide_data['cumulative_probable_statewide_deaths'],
'update_date':
update_date
}
try:
current_statewide_observation = StatewideTotalDate.objects.get(
scrape_date=today)
print('Updating existing statewide for {}'.format(today))
for k, v in base_record_obj.items():
setattr(current_statewide_observation, k, v)
current_statewide_observation.save()
except ObjectDoesNotExist:
try:
print('Creating 1st statewide record for {}'.format(today))
base_record_obj['scrape_date'] = today
current_statewide_observation = StatewideTotalDate(
**base_record_obj)
current_statewide_observation.save()
except Exception as e:
slack_latest('SCRAPER ERROR: {}'.format(e), '#robot-dojo')
raise
msg_output = ''
new_tests = current_statewide_observation.cumulative_completed_tests - yesterday_results.cumulative_completed_tests
print('Change found, composing statewide Slack message...')
msg_output += '*{}* cases total (change of *{}* today, *{}* newly reported, *{}* removed)\n'.format(
f'{current_statewide_observation.cumulative_positive_tests:,}',
self.change_sign(cases_daily_change),
f'{current_statewide_observation.cases_newly_reported:,}',
f'{current_statewide_observation.removed_cases:,}')
msg_output += '*{}* total hospitalizations (*{}* new admissions reported today)\n\n'.format(
f'{statewide_data["cumulative_hospitalized"]:,}',
self.change_sign(hospitalized_total_daily_change))
msg_output += '*{}* total icu (*{}* icu admissions reported today)\n\n'.format(
f'{statewide_data["cumulative_icu"]:,}',
self.change_sign(icu_total_daily_change))
final_msg = 'COVID scraper found updated data on the <https://www.health.state.mn.us/diseases/coronavirus/situation.html|MDH situation page>...\n\n' + msg_output
print(final_msg)
return final_msg
def update_county_records(self, county_data, update_date):
msg_output = ''
today = datetime.date.today()
bool_any_changes = False # Used to decide on Slack alert
for observation in county_data:
previous_county_observation = CountyTestDate.objects.filter(
county__name__iexact=observation['county'].strip(),
scrape_date__lt=today).order_by('-scrape_date').first()
if previous_county_observation:
previous_county_cases_total = previous_county_observation.cumulative_count
previous_county_deaths_total = previous_county_observation.cumulative_deaths
else:
previous_county_cases_total = 0
previous_county_deaths_total = 0
daily_cases = observation[
'cumulative_count'] - previous_county_cases_total
daily_deaths = observation[
'cumulative_deaths'] - previous_county_deaths_total
# Check if there is already an entry today
try:
county_observation = CountyTestDate.objects.get(
county__name__iexact=observation['county'].strip(),
scrape_date=today)
print('Updating {} County: {}'.format(
observation['county'], observation['cumulative_count']))
county_observation.update_date = update_date
county_observation.daily_total_cases = daily_cases
county_observation.cumulative_count = observation[
'cumulative_count']
county_observation.daily_deaths = daily_deaths
county_observation.cumulative_deaths = observation[
'cumulative_deaths']
county_observation.cumulative_confirmed_cases = observation[
'cumulative_confirmed_cases']
county_observation.cumulative_probable_cases = observation[
'cumulative_probable_cases']
county_observation.save()
except ObjectDoesNotExist:
try:
print('Creating 1st {} County record of day: {}'.format(
observation['county'],
observation['cumulative_count']))
bool_any_changes = True
county_observation = CountyTestDate(
county=County.objects.get(
name__iexact=observation['county'].strip()),
scrape_date=today,
update_date=update_date,
daily_total_cases=daily_cases,
cumulative_count=observation['cumulative_count'],
daily_deaths=daily_deaths,
cumulative_deaths=observation['cumulative_deaths'],
cumulative_confirmed_cases=observation[
'cumulative_confirmed_cases'],
cumulative_probable_cases=observation[
'cumulative_probable_cases'],
)
county_observation.save()
except Exception as e:
slack_latest('SCRAPER ERROR: {}'.format(e), '#robot-dojo')
raise
return bool_any_changes