def crawl_pdf(date, country):
if country == "Northern Ireland":
ym = dateparser.parse(date).strftime('%Y-%m')
dmy = dateparser.parse(date).strftime('%d.%m.%y')
pdf_url = "https://www.publichealth.hscni.net/sites/default/files/{}/COVID-19 Surveillance Bulletin {}.pdf".format(
ym, dmy)
local_pdf_file = "data/raw/Daily_bulletin_DoH_{}.pdf".format(date)
if not os.path.exists(local_pdf_file):
r = requests.get(pdf_url)
with open(local_pdf_file, "wb") as f:
f.write(r.content)
text = get_text_from_pdf(local_pdf_file)
results = parse_totals_pdf_text(country, text)
if results is None:
sys.stderr.write(
"Can't find numbers. Perhaps the page format has changed?\n")
sys.exit(1)
elif results["Date"] != date:
sys.stderr.write("Page is dated {}, but want {}\n".format(
results["Date"], date))
sys.exit(1)
print_totals(results)
#save_indicators(results)
save_indicators_to_sqlite(results)
daily_areas = parse_daily_areas_pdf(date, country, local_pdf_file)
if daily_areas is not None:
save_daily_areas(date, country, daily_areas)
save_daily_areas_to_sqlite(date, country, daily_areas)
def crawl_pdf(date, country, check_only):
if country == "Northern Ireland":
dt = dateparser.parse(date, date_formats=['%Y-%m-%d'], locales=["en-GB"])
ym = dt.strftime('%Y-%m')
dmy = dt.strftime('%d.%m.%y')
# the top-level page containing links to PDFs
html_url = "https://www.publichealth.hscni.net/publications/covid-19-surveillance-reports"
# the PDF itself
pdf_url = "https://www.publichealth.hscni.net/sites/default/files/{}/COVID-19 Surveillance Bulletin {}.pdf".format(ym, dmy)
local_pdf_file = "data/raw/Daily_bulletin_DoH_{}.pdf".format(date)
if not os.path.exists(local_pdf_file):
r = requests.get(html_url)
if "{}.pdf".format(dmy) not in r.text:
if check_only:
return DatasetUpdate.UPDATE_NOT_AVAILABLE
sys.stderr.write("Page is dated ?, but want {}\n".format(date))
sys.exit(1)
if check_only:
return DatasetUpdate.UPDATE_AVAILABLE
r = requests.get(pdf_url)
with open(local_pdf_file, "wb") as f:
f.write(r.content)
if check_only:
return DatasetUpdate.ALREADY_UPDATED
text = get_text_from_pdf(local_pdf_file)
results = parse_totals_pdf_text(country, text)
if results is None:
sys.stderr.write("Can't find numbers. Perhaps the page format has changed?\n")
sys.exit(1)
elif results["Date"] != date:
sys.stderr.write("Page is dated {}, but want {}\n".format(results["Date"], date))
sys.exit(1)
print_totals(results)
#save_indicators(results)
save_indicators_to_sqlite(results)
daily_areas = parse_daily_areas_pdf(date, country, local_pdf_file)
if daily_areas is not None:
save_daily_areas(date, country, daily_areas)
save_daily_areas_to_sqlite(date, country, daily_areas)
def crawl_html(date, country, check_only):
html_url = get_html_url(date, country)
local_html_file = "data/raw/coronavirus-covid-19-number-of-cases-in-{}-{}.html".format(
format_country(country), date)
save_html_file = False
try:
with open(local_html_file) as f:
html = f.read()
if check_only:
return DatasetUpdate.ALREADY_UPDATED
except FileNotFoundError:
r = requests.get(html_url)
html = r.text
save_html_file = True
results = parse_totals(country, html)
if results is None:
if check_only:
return DatasetUpdate.UPDATE_AVAILABLE
sys.stderr.write(
"Can't find numbers. Perhaps the page format has changed?\n")
sys.exit(1)
elif results["Date"] != date:
if check_only:
return DatasetUpdate.UPDATE_NOT_AVAILABLE
sys.stderr.write("Page is dated {}, but want {}\n".format(
results["Date"], date))
sys.exit(1)
if check_only:
return DatasetUpdate.UPDATE_AVAILABLE
daily_areas = parse_daily_areas(date, country, html)
print_totals(results)
#save_indicators(results)
save_indicators_to_sqlite(results)
if daily_areas is not None:
#save_daily_areas(date, country, daily_areas)
save_daily_areas_to_sqlite(date, country, daily_areas)
if save_html_file:
with open(local_html_file, "w") as f:
f.write(html)
